2024-09-13T21:38:47.172 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T21:38:47.207 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T21:38:47.456 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} email: ceph-qa@ceph.com first_in_suite: false job_id: '7904231' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: true mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_STRAY_DAEMON - CEPHADM_FAILED_DAEMON - CEPHADM_AGENT_DOWN log-only-match: - CEPHADM_ sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 workunit: branch: squid sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 3819 sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 sleep_before_teardown: 0 subset: 43/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 targets: smithi114.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPOXI3YeupnTZ2yKbxYYN9RY4dvyBqxyDpcl6rj72y8GQcUNTDukpem2EDM2mq8vTZUQHczcKuWIXdORKlFESHo= smithi121.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF4LSLuCrHEjYmnFRzbXmGqvULk6IMYQfB2XPjIdCXMgGZQmSsUD1yAtjRmETi18m5aq9+mUtgWic4QTru51VDk= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_21:08:03 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-13T21:38:47.457 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa; will attempt to use it 2024-09-13T21:38:47.458 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks 2024-09-13T21:38:47.458 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T21:38:47.460 INFO:teuthology.task.internal:Checking packages... 2024-09-13T21:38:47.484 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'ecab7be58963a5bd03910cd73eef66405d7fdd50' 2024-09-13T21:38:47.484 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T21:38:47.484 INFO:teuthology.packaging:ref: None 2024-09-13T21:38:47.484 INFO:teuthology.packaging:tag: None 2024-09-13T21:38:47.484 INFO:teuthology.packaging:branch: squid 2024-09-13T21:38:47.484 INFO:teuthology.packaging:sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:38:47.485 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-13T21:38:47.700 INFO:teuthology.task.internal:Found packages for ceph version 19.1.1-324.gecab7be5 2024-09-13T21:38:47.702 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T21:38:47.708 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T21:38:47.708 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T21:38:47.733 INFO:teuthology.task.internal:Saving configuration 2024-09-13T21:38:47.751 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T21:38:47.759 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T21:38:47.785 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi114.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 21:33:48.885104', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPOXI3YeupnTZ2yKbxYYN9RY4dvyBqxyDpcl6rj72y8GQcUNTDukpem2EDM2mq8vTZUQHczcKuWIXdORKlFESHo='} 2024-09-13T21:38:47.811 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi121.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 21:33:48.887015', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF4LSLuCrHEjYmnFRzbXmGqvULk6IMYQfB2XPjIdCXMgGZQmSsUD1yAtjRmETi18m5aq9+mUtgWic4QTru51VDk='} 2024-09-13T21:38:47.811 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T21:38:47.817 INFO:teuthology.task.internal:roles: ubuntu@smithi114.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T21:38:47.817 INFO:teuthology.task.internal:roles: ubuntu@smithi121.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T21:38:47.818 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T21:38:47.883 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb5f66bbac0>, signals=[15]) 2024-09-13T21:38:47.884 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T21:38:47.890 INFO:teuthology.task.internal:Opening connections... 2024-09-13T21:38:47.890 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi114.front.sepia.ceph.com 2024-09-13T21:38:47.892 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:38:47.977 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi121.front.sepia.ceph.com 2024-09-13T21:38:47.978 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:38:48.062 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T21:38:48.068 DEBUG:teuthology.orchestra.run.smithi114:> uname -m 2024-09-13T21:38:48.087 INFO:teuthology.orchestra.run.smithi114.stdout:x86_64 2024-09-13T21:38:48.087 DEBUG:teuthology.orchestra.run.smithi114:> cat /etc/os-release 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:NAME="CentOS Stream" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:VERSION="9" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:ID="centos" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:ID_LIKE="rhel fedora" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:VERSION_ID="9" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:PLATFORM_ID="platform:el9" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:ANSI_COLOR="0;31" 2024-09-13T21:38:48.146 INFO:teuthology.orchestra.run.smithi114.stdout:LOGO="fedora-logo-icon" 2024-09-13T21:38:48.147 INFO:teuthology.orchestra.run.smithi114.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T21:38:48.147 INFO:teuthology.orchestra.run.smithi114.stdout:HOME_URL="https://centos.org/" 2024-09-13T21:38:48.147 INFO:teuthology.orchestra.run.smithi114.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T21:38:48.147 INFO:teuthology.orchestra.run.smithi114.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T21:38:48.147 INFO:teuthology.orchestra.run.smithi114.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T21:38:48.147 INFO:teuthology.lock.ops:Updating smithi114.front.sepia.ceph.com on lock server 2024-09-13T21:38:48.182 DEBUG:teuthology.orchestra.run.smithi121:> uname -m 2024-09-13T21:38:48.201 INFO:teuthology.orchestra.run.smithi121.stdout:x86_64 2024-09-13T21:38:48.201 DEBUG:teuthology.orchestra.run.smithi121:> cat /etc/os-release 2024-09-13T21:38:48.258 INFO:teuthology.orchestra.run.smithi121.stdout:NAME="CentOS Stream" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION="9" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:ID="centos" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:ID_LIKE="rhel fedora" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION_ID="9" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:PLATFORM_ID="platform:el9" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:ANSI_COLOR="0;31" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:LOGO="fedora-logo-icon" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:HOME_URL="https://centos.org/" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T21:38:48.259 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T21:38:48.260 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T21:38:48.260 INFO:teuthology.lock.ops:Updating smithi121.front.sepia.ceph.com on lock server 2024-09-13T21:38:48.290 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T21:38:48.298 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T21:38:48.303 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T21:38:48.304 DEBUG:teuthology.orchestra.run.smithi114:> test '!' -e /home/ubuntu/cephtest 2024-09-13T21:38:48.306 DEBUG:teuthology.orchestra.run.smithi121:> test '!' -e /home/ubuntu/cephtest 2024-09-13T21:38:48.323 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T21:38:48.329 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T21:38:48.329 DEBUG:teuthology.orchestra.run.smithi114:> test -z $(ls -A /var/lib/ceph) 2024-09-13T21:38:48.364 DEBUG:teuthology.orchestra.run.smithi121:> test -z $(ls -A /var/lib/ceph) 2024-09-13T21:38:48.395 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T21:38:48.461 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T21:38:48.485 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T21:38:48.485 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T21:38:48.485 DEBUG:teuthology.orchestra.run.smithi114:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T21:38:48.486 DEBUG:teuthology.orchestra.run.smithi121:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T21:38:48.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:38:48.504 DEBUG:teuthology.orchestra.run.smithi121:> uname -r 2024-09-13T21:38:48.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:38:48.506 DEBUG:teuthology.orchestra.run.smithi114:> uname -r 2024-09-13T21:38:48.561 INFO:teuthology.orchestra.run.smithi121.stdout:5.14.0-503.el9.x86_64 2024-09-13T21:38:48.561 INFO:teuthology.task.kernel:Running kernel on smithi121: 5.14.0-503.el9.x86_64 2024-09-13T21:38:48.561 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum install -y kernel 2024-09-13T21:38:48.564 INFO:teuthology.orchestra.run.smithi114.stdout:5.14.0-503.el9.x86_64 2024-09-13T21:38:48.564 INFO:teuthology.task.kernel:Running kernel on smithi114: 5.14.0-503.el9.x86_64 2024-09-13T21:38:48.564 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum install -y kernel 2024-09-13T21:38:50.822 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.2 MB 00:00 2024-09-13T21:38:51.028 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.2 MB 00:00 2024-09-13T21:38:52.506 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - AppStream 36 MB/s | 20 MB 00:00 2024-09-13T21:38:55.437 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - AppStream 6.2 MB/s | 20 MB 00:03 2024-09-13T21:38:56.873 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.5 MB 00:00 2024-09-13T21:38:58.399 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 19 kB 00:00 2024-09-13T21:38:59.278 INFO:teuthology.orchestra.run.smithi114.stdout:Extra Packages for Enterprise Linux 30 MB/s | 23 MB 00:00 2024-09-13T21:39:01.786 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - CRB 2.6 MB/s | 6.5 MB 00:02 2024-09-13T21:39:03.429 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 9 - Extras packages 60 kB/s | 19 kB 00:00 2024-09-13T21:39:04.199 INFO:teuthology.orchestra.run.smithi121.stdout:Extra Packages for Enterprise Linux 35 MB/s | 23 MB 00:00 2024-09-13T21:39:04.701 INFO:teuthology.orchestra.run.smithi114.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-13T21:39:06.749 INFO:teuthology.orchestra.run.smithi114.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T21:39:06.749 INFO:teuthology.orchestra.run.smithi114.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T21:39:06.859 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-09-13T21:39:06.864 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-09-13T21:39:06.864 INFO:teuthology.orchestra.run.smithi114.stdout: Package Architecture Version Repository Size 2024-09-13T21:39:06.864 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-09-13T21:39:06.864 INFO:teuthology.orchestra.run.smithi114.stdout:Installing: 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout:Installing dependencies: 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout:Install 4 Packages 2024-09-13T21:39:06.865 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:39:06.866 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 79 M 2024-09-13T21:39:06.866 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 126 M 2024-09-13T21:39:06.866 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-09-13T21:39:07.035 INFO:teuthology.orchestra.run.smithi114.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 189 kB/s | 23 kB 00:00 2024-09-13T21:39:07.652 INFO:teuthology.orchestra.run.smithi114.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 21 MB/s | 16 MB 00:00 2024-09-13T21:39:08.252 INFO:teuthology.orchestra.run.smithi114.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 23 MB/s | 29 MB 00:01 2024-09-13T21:39:08.561 INFO:teuthology.orchestra.run.smithi114.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 21 MB/s | 35 MB 00:01 2024-09-13T21:39:08.561 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-09-13T21:39:08.562 INFO:teuthology.orchestra.run.smithi114.stdout:Total 46 MB/s | 79 MB 00:01 2024-09-13T21:39:09.247 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-09-13T21:39:09.598 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-09-13T21:39:09.598 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-09-13T21:39:09.705 INFO:teuthology.orchestra.run.smithi121.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-13T21:39:10.554 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-09-13T21:39:10.554 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-09-13T21:39:11.711 INFO:teuthology.orchestra.run.smithi121.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T21:39:11.712 INFO:teuthology.orchestra.run.smithi121.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T21:39:11.796 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2024-09-13T21:39:11.806 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-09-13T21:39:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: Package Architecture Version Repository Size 2024-09-13T21:39:11.806 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-09-13T21:39:11.806 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2024-09-13T21:39:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout:Install 4 Packages 2024-09-13T21:39:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:39:11.808 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 79 M 2024-09-13T21:39:11.808 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 126 M 2024-09-13T21:39:11.808 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2024-09-13T21:39:11.932 INFO:teuthology.orchestra.run.smithi121.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 301 kB/s | 23 kB 00:00 2024-09-13T21:39:12.994 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-09-13T21:39:13.638 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:39:13.711 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:39:14.798 INFO:teuthology.orchestra.run.smithi121.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 5.3 MB/s | 16 MB 00:02 2024-09-13T21:39:16.501 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:39:16.638 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:39:16.989 INFO:teuthology.orchestra.run.smithi121.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 5.6 MB/s | 29 MB 00:05 2024-09-13T21:39:18.531 INFO:teuthology.orchestra.run.smithi121.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 5.2 MB/s | 35 MB 00:06 2024-09-13T21:39:18.531 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2024-09-13T21:39:18.532 INFO:teuthology.orchestra.run.smithi121.stdout:Total 12 MB/s | 79 MB 00:06 2024-09-13T21:39:19.220 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2024-09-13T21:39:19.567 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2024-09-13T21:39:19.567 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2024-09-13T21:39:20.510 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2024-09-13T21:39:20.511 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2024-09-13T21:39:21.481 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:39:21.622 INFO:teuthology.orchestra.run.smithi114.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:39:22.949 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2024-09-13T21:39:23.624 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:39:23.732 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:39:26.445 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:39:26.518 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:39:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:39:31.481 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:39:31.518 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:39:36.379 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:04.861 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:04.900 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:05.163 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:05.163 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:40:05.163 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:40:05.163 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:40:06.135 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout:Installed: 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:40:06.136 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-09-13T21:40:06.480 DEBUG:teuthology.orchestra.run.smithi114:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:40:06.569 INFO:teuthology.orchestra.run.smithi114.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:40:06.570 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:40:06.570 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T21:40:06.570 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T21:40:06.570 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T21:40:06.571 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T21:40:06.571 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi114.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T21:40:06.571 DEBUG:teuthology.orchestra.run.smithi114:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:40:06.629 INFO:teuthology.orchestra.run.smithi114.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:40:06.629 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:40:06.630 DEBUG:teuthology.orchestra.run.smithi114:> sudo rpm -qi grub2-tools 2024-09-13T21:40:06.661 INFO:teuthology.orchestra.run.smithi114.stdout:Name : grub2-tools 2024-09-13T21:40:06.661 INFO:teuthology.orchestra.run.smithi114.stdout:Epoch : 1 2024-09-13T21:40:06.661 INFO:teuthology.orchestra.run.smithi114.stdout:Version : 2.06 2024-09-13T21:40:06.661 INFO:teuthology.orchestra.run.smithi114.stdout:Release : 82.el9 2024-09-13T21:40:06.661 INFO:teuthology.orchestra.run.smithi114.stdout:Architecture: x86_64 2024-09-13T21:40:06.661 INFO:teuthology.orchestra.run.smithi114.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Group : Unspecified 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Size : 8274670 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:License : GPLv3+ 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Packager : builder@centos.org 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Vendor : CentOS 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Summary : Support tools for GRUB. 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout:Description : 2024-09-13T21:40:06.662 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:40:06.663 INFO:teuthology.orchestra.run.smithi114.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T21:40:06.663 INFO:teuthology.orchestra.run.smithi114.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T21:40:06.663 INFO:teuthology.orchestra.run.smithi114.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T21:40:06.663 INFO:teuthology.orchestra.run.smithi114.stdout:hardware devices. 2024-09-13T21:40:06.663 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:40:06.663 INFO:teuthology.orchestra.run.smithi114.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T21:40:06.664 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T21:40:06.664 INFO:teuthology.task.kernel:Updating grub on smithi114 to boot 5.14.0-505.el9.x86_64 2024-09-13T21:40:06.664 DEBUG:teuthology.orchestra.run.smithi114:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T21:40:07.334 INFO:teuthology.orchestra.run.smithi114.stderr:Generating grub configuration file ... 2024-09-13T21:40:08.706 INFO:teuthology.orchestra.run.smithi114.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T21:40:08.746 INFO:teuthology.orchestra.run.smithi114.stderr:done 2024-09-13T21:40:08.748 DEBUG:teuthology.orchestra.run.smithi114:> mktemp 2024-09-13T21:40:08.763 INFO:teuthology.orchestra.run.smithi114.stdout:/tmp/tmp.mxIGs9vtJV 2024-09-13T21:40:08.763 DEBUG:teuthology.orchestra.run.smithi114:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.mxIGs9vtJV 2024-09-13T21:40:08.829 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod 0666 /tmp/tmp.mxIGs9vtJV 2024-09-13T21:40:08.964 DEBUG:teuthology.orchestra.remote:smithi114:/tmp/tmp.mxIGs9vtJV is 6KB 2024-09-13T21:40:09.013 DEBUG:teuthology.orchestra.run.smithi114:> rm -fr /tmp/tmp.mxIGs9vtJV 2024-09-13T21:40:09.028 DEBUG:teuthology.orchestra.run.smithi114:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T21:40:09.098 INFO:teuthology.orchestra.run.smithi114.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T21:40:09.099 INFO:teuthology.orchestra.run.smithi114.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T21:40:09.099 INFO:teuthology.orchestra.run.smithi114.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T21:40:09.099 INFO:teuthology.orchestra.run.smithi114.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T21:40:09.100 DEBUG:teuthology.orchestra.run.smithi114:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T21:40:09.250 DEBUG:teuthology.orchestra.run.smithi114:> sudo shutdown -r now 2024-09-13T21:40:14.660 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:14.700 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:15.049 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:15.049 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:40:15.050 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:40:15.050 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T21:40:15.933 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:40:15.934 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2024-09-13T21:40:16.273 DEBUG:teuthology.orchestra.run.smithi121:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:40:16.400 INFO:teuthology.orchestra.run.smithi121.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:40:16.400 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:40:16.400 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T21:40:16.401 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T21:40:16.401 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T21:40:16.401 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T21:40:16.401 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi121.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T21:40:16.401 DEBUG:teuthology.orchestra.run.smithi121:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:40:16.460 INFO:teuthology.orchestra.run.smithi121.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:40:16.461 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:40:16.461 DEBUG:teuthology.orchestra.run.smithi121:> sudo rpm -qi grub2-tools 2024-09-13T21:40:16.496 INFO:teuthology.orchestra.run.smithi121.stdout:Name : grub2-tools 2024-09-13T21:40:16.496 INFO:teuthology.orchestra.run.smithi121.stdout:Epoch : 1 2024-09-13T21:40:16.496 INFO:teuthology.orchestra.run.smithi121.stdout:Version : 2.06 2024-09-13T21:40:16.496 INFO:teuthology.orchestra.run.smithi121.stdout:Release : 82.el9 2024-09-13T21:40:16.496 INFO:teuthology.orchestra.run.smithi121.stdout:Architecture: x86_64 2024-09-13T21:40:16.496 INFO:teuthology.orchestra.run.smithi121.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T21:40:16.496 INFO:teuthology.orchestra.run.smithi121.stdout:Group : Unspecified 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Size : 8274670 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:License : GPLv3+ 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Packager : builder@centos.org 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Vendor : CentOS 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Summary : Support tools for GRUB. 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:Description : 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:40:16.497 INFO:teuthology.orchestra.run.smithi121.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T21:40:16.498 INFO:teuthology.orchestra.run.smithi121.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T21:40:16.498 INFO:teuthology.orchestra.run.smithi121.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T21:40:16.498 INFO:teuthology.orchestra.run.smithi121.stdout:hardware devices. 2024-09-13T21:40:16.498 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:40:16.498 INFO:teuthology.orchestra.run.smithi121.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T21:40:16.499 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T21:40:16.499 INFO:teuthology.task.kernel:Updating grub on smithi121 to boot 5.14.0-505.el9.x86_64 2024-09-13T21:40:16.499 DEBUG:teuthology.orchestra.run.smithi121:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T21:40:17.141 INFO:teuthology.orchestra.run.smithi121.stderr:Generating grub configuration file ... 2024-09-13T21:40:18.450 INFO:teuthology.orchestra.run.smithi121.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T21:40:18.484 INFO:teuthology.orchestra.run.smithi121.stderr:done 2024-09-13T21:40:18.486 DEBUG:teuthology.orchestra.run.smithi121:> mktemp 2024-09-13T21:40:18.502 INFO:teuthology.orchestra.run.smithi121.stdout:/tmp/tmp.VLbTS3t71a 2024-09-13T21:40:18.502 DEBUG:teuthology.orchestra.run.smithi121:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.VLbTS3t71a 2024-09-13T21:40:18.570 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 0666 /tmp/tmp.VLbTS3t71a 2024-09-13T21:40:18.718 DEBUG:teuthology.orchestra.remote:smithi121:/tmp/tmp.VLbTS3t71a is 6KB 2024-09-13T21:40:18.767 DEBUG:teuthology.orchestra.run.smithi121:> rm -fr /tmp/tmp.VLbTS3t71a 2024-09-13T21:40:18.781 DEBUG:teuthology.orchestra.run.smithi121:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T21:40:18.844 INFO:teuthology.orchestra.run.smithi121.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T21:40:18.844 INFO:teuthology.orchestra.run.smithi121.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T21:40:18.845 INFO:teuthology.orchestra.run.smithi121.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T21:40:18.845 INFO:teuthology.orchestra.run.smithi121.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T21:40:18.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T21:40:19.000 DEBUG:teuthology.orchestra.run.smithi121:> sudo shutdown -r now 2024-09-13T21:40:39.273 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T21:40:39.273 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-09-13T21:40:39.274 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:40:49.010 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T21:40:49.011 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-09-13T21:40:49.012 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:40:57.789 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.114 2024-09-13T21:41:06.797 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-09-13T21:41:06.798 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:41:07.517 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.121 2024-09-13T21:41:09.885 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.114 2024-09-13T21:41:16.525 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-09-13T21:41:16.526 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:41:19.613 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.121 2024-09-13T21:41:21.888 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-09-13T21:41:21.888 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:41:31.623 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-09-13T21:41:31.624 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:41:53.679 DEBUG:teuthology.orchestra.run.smithi114:> true 2024-09-13T21:41:54.040 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-09-13T21:41:54.040 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T21:41:54.040 DEBUG:teuthology.orchestra.run.smithi114:> uname -r 2024-09-13T21:41:54.055 INFO:teuthology.orchestra.run.smithi114.stdout:5.14.0-505.el9.x86_64 2024-09-13T21:41:54.055 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T21:41:54.055 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T21:41:54.055 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T21:41:55.056 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T21:41:55.056 DEBUG:teuthology.orchestra.run.smithi114:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T21:41:55.209 INFO:teuthology.orchestra.run.smithi114.stdout:ttyS1 2024-09-13T21:41:55.232 DEBUG:teuthology.parallel:result is None 2024-09-13T21:42:04.132 DEBUG:teuthology.orchestra.run.smithi121:> true 2024-09-13T21:42:04.478 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-09-13T21:42:04.479 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T21:42:04.479 DEBUG:teuthology.orchestra.run.smithi121:> uname -r 2024-09-13T21:42:04.493 INFO:teuthology.orchestra.run.smithi121.stdout:5.14.0-505.el9.x86_64 2024-09-13T21:42:04.493 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T21:42:04.494 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T21:42:04.494 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T21:42:05.495 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T21:42:05.495 DEBUG:teuthology.orchestra.run.smithi121:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T21:42:05.558 INFO:teuthology.orchestra.run.smithi121.stdout:ttyS1 2024-09-13T21:42:05.579 DEBUG:teuthology.parallel:result is None 2024-09-13T21:42:05.579 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T21:42:05.587 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T21:42:05.587 DEBUG:teuthology.orchestra.run.smithi114:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T21:42:05.590 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T21:42:05.608 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T21:42:05.613 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T21:42:05.623 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T21:42:05.623 DEBUG:teuthology.orchestra.run.smithi114:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T21:42:05.649 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T21:42:05.757 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T21:42:05.763 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T21:42:05.763 DEBUG:teuthology.orchestra.run.smithi114:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T21:42:05.766 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T21:42:05.802 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:42:05.808 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:42:05.814 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:42:05.819 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:42:05.821 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T21:42:05.828 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T21:42:05.828 DEBUG:teuthology.orchestra.run.smithi114:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T21:42:05.858 DEBUG:teuthology.orchestra.run.smithi121:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T21:42:05.887 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T21:42:05.896 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T21:42:05.896 DEBUG:teuthology.orchestra.run.smithi114:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T21:42:05.927 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T21:42:05.943 DEBUG:teuthology.orchestra.run.smithi114:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:42:06.034 DEBUG:teuthology.orchestra.run.smithi114:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:42:06.083 DEBUG:teuthology.orchestra.run.smithi114:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:42:06.174 DEBUG:teuthology.orchestra.run.smithi114:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:42:06.209 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:42:06.209 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T21:42:06.275 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:42:06.328 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:42:06.375 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:42:06.465 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:42:06.502 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-09-13T21:42:06.502 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T21:42:06.569 DEBUG:teuthology.orchestra.run.smithi114:> sudo service rsyslog restart 2024-09-13T21:42:06.572 DEBUG:teuthology.orchestra.run.smithi121:> sudo service rsyslog restart 2024-09-13T21:42:06.648 INFO:teuthology.orchestra.run.smithi114.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T21:42:06.658 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T21:42:07.167 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T21:42:07.174 INFO:teuthology.task.internal:Starting timer... 2024-09-13T21:42:07.174 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T21:42:07.190 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T21:42:07.198 DEBUG:teuthology.orchestra.run.smithi114:> sudo service auditd rotate 2024-09-13T21:42:07.240 INFO:teuthology.orchestra.run.smithi114.stdout:Rotating logs: 2024-09-13T21:42:07.242 DEBUG:teuthology.orchestra.run.smithi121:> sudo service auditd rotate 2024-09-13T21:42:07.287 INFO:teuthology.orchestra.run.smithi121.stdout:Rotating logs: 2024-09-13T21:42:07.289 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T21:42:07.290 DEBUG:teuthology.orchestra.run.smithi114:> /usr/sbin/getenforce 2024-09-13T21:42:07.316 INFO:teuthology.orchestra.run.smithi114.stdout:Permissive 2024-09-13T21:42:07.317 DEBUG:teuthology.orchestra.run.smithi121:> /usr/sbin/getenforce 2024-09-13T21:42:07.353 INFO:teuthology.orchestra.run.smithi121.stdout:Permissive 2024-09-13T21:42:07.353 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi114.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi121.front.sepia.ceph.com': 'permissive'} 2024-09-13T21:42:07.353 DEBUG:teuthology.orchestra.run.smithi114:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T21:42:07.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:42:07.384 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T21:42:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:42:07.422 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T21:42:07.423 DEBUG:teuthology.orchestra.run.smithi114:> sudo /usr/sbin/setenforce permissive 2024-09-13T21:42:07.466 DEBUG:teuthology.orchestra.run.smithi121:> sudo /usr/sbin/setenforce permissive 2024-09-13T21:42:07.503 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T21:42:07.513 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T21:42:07.525 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T21:42:07.526 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi114.front.sepia.ceph.com,smithi121.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T21:45:52.779 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi114.front.sepia.ceph.com'), Remote(name='ubuntu@smithi121.front.sepia.ceph.com')] 2024-09-13T21:45:52.781 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-09-13T21:45:52.782 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:45:52.870 DEBUG:teuthology.orchestra.run.smithi114:> true 2024-09-13T21:45:52.952 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-09-13T21:45:52.952 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-09-13T21:45:52.953 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:45:53.035 DEBUG:teuthology.orchestra.run.smithi121:> true 2024-09-13T21:45:53.075 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-09-13T21:45:53.075 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T21:45:53.085 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T21:45:53.086 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T21:45:53.086 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T21:45:53.088 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T21:45:53.088 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T21:45:53.115 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T21:45:53.128 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T21:45:53.145 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T21:45:53.152 INFO:teuthology.orchestra.run.smithi114.stderr:sudo: ntpd: command not found 2024-09-13T21:45:53.159 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T21:45:53.163 INFO:teuthology.orchestra.run.smithi114.stdout:506 Cannot talk to daemon 2024-09-13T21:45:53.177 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T21:45:53.183 INFO:teuthology.orchestra.run.smithi121.stderr:sudo: ntpd: command not found 2024-09-13T21:45:53.191 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T21:45:53.194 INFO:teuthology.orchestra.run.smithi121.stdout:506 Cannot talk to daemon 2024-09-13T21:45:53.208 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T21:45:53.222 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T21:45:53.254 INFO:teuthology.orchestra.run.smithi114.stderr:bash: line 1: ntpq: command not found 2024-09-13T21:45:53.257 INFO:teuthology.orchestra.run.smithi114.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T21:45:53.257 INFO:teuthology.orchestra.run.smithi114.stdout:=============================================================================== 2024-09-13T21:45:53.257 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.257 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.257 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.257 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.279 INFO:teuthology.orchestra.run.smithi121.stderr:bash: line 1: ntpq: command not found 2024-09-13T21:45:53.282 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T21:45:53.282 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2024-09-13T21:45:53.282 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.282 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.282 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.282 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:45:53.283 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T21:45:53.373 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': True}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_STRAY_DAEMON', 'CEPHADM_FAILED_DAEMON', 'CEPHADM_AGENT_DOWN'], 'log-only-match': ['CEPHADM_'], 'sha1': 'ecab7be58963a5bd03910cd73eef66405d7fdd50'} 2024-09-13T21:45:53.373 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-13T21:45:53.374 INFO:tasks.cephadm:Cluster fsid is 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:45:53.374 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T21:45:53.374 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.114', 'mon.c': '[v2:172.21.15.114:3301,v1:172.21.15.114:6790]', 'mon.b': '172.21.15.121'} 2024-09-13T21:45:53.374 INFO:tasks.cephadm:First mon is mon.a on smithi114 2024-09-13T21:45:53.374 INFO:tasks.cephadm:First mgr is y 2024-09-13T21:45:53.375 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T21:45:53.375 DEBUG:teuthology.orchestra.run.smithi114:> sudo hostname $(hostname -s) 2024-09-13T21:45:53.400 DEBUG:teuthology.orchestra.run.smithi121:> sudo hostname $(hostname -s) 2024-09-13T21:45:53.426 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-13T21:45:53.427 DEBUG:teuthology.orchestra.run.smithi114:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T21:45:53.620 INFO:teuthology.orchestra.run.smithi114.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 21:45 /home/ubuntu/cephtest/cephadm 2024-09-13T21:45:53.620 DEBUG:teuthology.orchestra.run.smithi121:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T21:45:53.744 INFO:teuthology.orchestra.run.smithi121.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 21:45 /home/ubuntu/cephtest/cephadm 2024-09-13T21:45:53.745 DEBUG:teuthology.orchestra.run.smithi114:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T21:45:53.761 DEBUG:teuthology.orchestra.run.smithi121:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T21:45:53.787 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-13T21:45:53.787 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T21:45:53.804 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T21:45:55.362 INFO:teuthology.orchestra.run.smithi121.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:45:55.392 INFO:teuthology.orchestra.run.smithi114.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:46:07.290 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T21:46:07.290 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T21:46:07.290 INFO:teuthology.orchestra.run.smithi114.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T21:46:07.290 INFO:teuthology.orchestra.run.smithi114.stdout: "repo_digests": [ 2024-09-13T21:46:07.291 INFO:teuthology.orchestra.run.smithi114.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T21:46:07.291 INFO:teuthology.orchestra.run.smithi114.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T21:46:07.291 INFO:teuthology.orchestra.run.smithi114.stdout: ] 2024-09-13T21:46:07.291 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T21:46:07.541 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-09-13T21:46:07.541 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T21:46:07.541 INFO:teuthology.orchestra.run.smithi121.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T21:46:07.541 INFO:teuthology.orchestra.run.smithi121.stdout: "repo_digests": [ 2024-09-13T21:46:07.541 INFO:teuthology.orchestra.run.smithi121.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T21:46:07.542 INFO:teuthology.orchestra.run.smithi121.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T21:46:07.542 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2024-09-13T21:46:07.542 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-09-13T21:46:07.560 DEBUG:teuthology.orchestra.run.smithi114:> sudo mkdir -p /etc/ceph 2024-09-13T21:46:07.587 DEBUG:teuthology.orchestra.run.smithi121:> sudo mkdir -p /etc/ceph 2024-09-13T21:46:07.616 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod 777 /etc/ceph 2024-09-13T21:46:07.652 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 777 /etc/ceph 2024-09-13T21:46:07.681 INFO:tasks.cephadm:Writing seed config... 2024-09-13T21:46:07.682 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-13T21:46:07.682 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T21:46:07.682 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T21:46:07.682 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = True 2024-09-13T21:46:07.683 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T21:46:07.683 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T21:46:07.683 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T21:46:07.683 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T21:46:07.683 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T21:46:07.683 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:46:07.683 DEBUG:teuthology.orchestra.run.smithi114:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T21:46:07.707 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 8ccc9cd8-7219-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = True [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T21:46:07.707 DEBUG:teuthology.orchestra.run.smithi114:mon.a> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T21:46:07.750 DEBUG:teuthology.orchestra.run.smithi114:mgr.y> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T21:46:07.792 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T21:46:07.792 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.114 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:46:07.981 INFO:teuthology.orchestra.run.smithi114.stderr:-------------------------------------------------------------------------------- 2024-09-13T21:46:07.981 INFO:teuthology.orchestra.run.smithi114.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '8ccc9cd8-7219-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.114', '--skip-admin-label'] 2024-09-13T21:46:08.002 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: 5.2.2 2024-09-13T21:46:08.005 INFO:teuthology.orchestra.run.smithi114.stderr:Verifying podman|docker is present... 2024-09-13T21:46:08.029 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: 5.2.2 2024-09-13T21:46:08.031 INFO:teuthology.orchestra.run.smithi114.stderr:Verifying lvm2 is present... 2024-09-13T21:46:08.031 INFO:teuthology.orchestra.run.smithi114.stderr:Verifying time synchronization is in place... 2024-09-13T21:46:08.038 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T21:46:08.045 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: inactive 2024-09-13T21:46:08.051 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: enabled 2024-09-13T21:46:08.057 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: active 2024-09-13T21:46:08.058 INFO:teuthology.orchestra.run.smithi114.stderr:Unit chronyd.service is enabled and running 2024-09-13T21:46:08.058 INFO:teuthology.orchestra.run.smithi114.stderr:Repeating the final host check... 2024-09-13T21:46:08.078 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: 5.2.2 2024-09-13T21:46:08.080 INFO:teuthology.orchestra.run.smithi114.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-13T21:46:08.080 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl is present 2024-09-13T21:46:08.080 INFO:teuthology.orchestra.run.smithi114.stderr:lvcreate is present 2024-09-13T21:46:08.086 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T21:46:08.092 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: inactive 2024-09-13T21:46:08.099 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: enabled 2024-09-13T21:46:08.105 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: active 2024-09-13T21:46:08.106 INFO:teuthology.orchestra.run.smithi114.stderr:Unit chronyd.service is enabled and running 2024-09-13T21:46:08.107 INFO:teuthology.orchestra.run.smithi114.stderr:Host looks OK 2024-09-13T21:46:08.107 INFO:teuthology.orchestra.run.smithi114.stderr:Cluster fsid: 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:08.107 INFO:teuthology.orchestra.run.smithi114.stderr:Acquiring lock 140709155299968 on /run/cephadm/8ccc9cd8-7219-11ef-bceb-c7b262605968.lock 2024-09-13T21:46:08.107 INFO:teuthology.orchestra.run.smithi114.stderr:Lock 140709155299968 acquired on /run/cephadm/8ccc9cd8-7219-11ef-bceb-c7b262605968.lock 2024-09-13T21:46:08.107 INFO:teuthology.orchestra.run.smithi114.stderr:Verifying IP 172.21.15.114 port 3300 ... 2024-09-13T21:46:08.107 INFO:teuthology.orchestra.run.smithi114.stderr:Verifying IP 172.21.15.114 port 6789 ... 2024-09-13T21:46:08.107 INFO:teuthology.orchestra.run.smithi114.stderr:Base mon IP is 172.21.15.114, final addrv is [v2:172.21.15.114:3300,v1:172.21.15.114:6789] 2024-09-13T21:46:08.109 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.114 metric 100 2024-09-13T21:46:08.109 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.114 metric 100 2024-09-13T21:46:08.112 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T21:46:08.112 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-13T21:46:08.112 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-13T21:46:08.114 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T21:46:08.114 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-13T21:46:08.114 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T21:46:08.115 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-13T21:46:08.115 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:cb63/64 scope link noprefixroute 2024-09-13T21:46:08.115 INFO:teuthology.orchestra.run.smithi114.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T21:46:08.115 INFO:teuthology.orchestra.run.smithi114.stderr:Mon IP `172.21.15.114` is in CIDR network `172.21.0.0/20` 2024-09-13T21:46:08.115 INFO:teuthology.orchestra.run.smithi114.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T21:46:08.116 INFO:teuthology.orchestra.run.smithi114.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:46:08.215 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:46:08.943 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Getting image source signatures 2024-09-13T21:46:08.943 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-13T21:46:08.943 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-13T21:46:08.943 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-13T21:46:08.943 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-13T21:46:08.944 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-13T21:46:08.944 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T21:46:09.102 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: Writing manifest to image destination 2024-09-13T21:46:09.256 INFO:teuthology.orchestra.run.smithi114.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T21:46:09.666 INFO:teuthology.orchestra.run.smithi114.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T21:46:10.112 INFO:teuthology.orchestra.run.smithi114.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T21:46:10.112 INFO:teuthology.orchestra.run.smithi114.stderr:Extracting ceph user uid/gid from container image... 2024-09-13T21:46:10.478 INFO:teuthology.orchestra.run.smithi114.stderr:stat: 167 167 2024-09-13T21:46:11.000 INFO:teuthology.orchestra.run.smithi114.stderr:Creating initial keys... 2024-09-13T21:46:11.409 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph-authtool: AQCjsuRm6OI/EhAADT1qBZHZRthcEQ8DcpxB8A== 2024-09-13T21:46:12.380 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph-authtool: AQCksuRm248zERAANJhxPgvDk2IAVLn/DvNrwQ== 2024-09-13T21:46:13.202 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph-authtool: AQClsuRmAf4EBxAA76zE8u9de7Uenv82feOHrw== 2024-09-13T21:46:13.676 INFO:teuthology.orchestra.run.smithi114.stderr:Creating initial monmap... 2024-09-13T21:46:14.099 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T21:46:14.100 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-13T21:46:14.100 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:14.100 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T21:46:14.581 INFO:teuthology.orchestra.run.smithi114.stderr:monmaptool for a [v2:172.21.15.114:3300,v1:172.21.15.114:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T21:46:14.581 INFO:teuthology.orchestra.run.smithi114.stderr:setting min_mon_release = octopus 2024-09-13T21:46:14.581 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/monmaptool: set fsid to 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:14.581 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T21:46:14.582 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:46:14.582 INFO:teuthology.orchestra.run.smithi114.stderr:Creating mon... 2024-09-13T21:46:15.453 INFO:teuthology.orchestra.run.smithi114.stderr:create mon.a on 2024-09-13T21:46:15.654 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T21:46:15.798 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968.target. 2024-09-13T21:46:15.798 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968.target. 2024-09-13T21:46:16.253 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Failed to reset failed state of unit ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T21:46:16.262 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Created symlink /etc/systemd/system/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968.target.wants/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@.service. 2024-09-13T21:46:16.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:16 smithi114 podman[23364]: 2024-09-13 21:46:16.65892022 +0000 UTC m=+0.018814573 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T21:46:17.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:17 smithi114 podman[23364]: 2024-09-13 21:46:17.518594012 +0000 UTC m=+0.878488354 container create 677889c5c4dd50d920301c6e69e69a7d3e0a95882a87c374da5ae71fbe1dccb1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, version=8, RELEASE=HEAD, architecture=x86_64, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , release=754, ceph=True, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T21:46:19.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:19 smithi114 podman[23364]: 2024-09-13 21:46:19.495899097 +0000 UTC m=+2.855793446 container init 677889c5c4dd50d920301c6e69e69a7d3e0a95882a87c374da5ae71fbe1dccb1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, version=8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.tags=base centos centos-stream, vcs-type=git, distribution-scope=public, RELEASE=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T21:46:19.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:19 smithi114 podman[23364]: 2024-09-13 21:46:19.499556281 +0000 UTC m=+2.859450628 container start 677889c5c4dd50d920301c6e69e69a7d3e0a95882a87c374da5ae71fbe1dccb1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, release=754, version=8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-type=git, ceph=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T21:46:19.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:19 smithi114 ceph-mon[23390]: mkfs 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:19.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:19 smithi114 ceph-mon[23390]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T21:46:20.438 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: disabled 2024-09-13T21:46:20.446 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: inactive 2024-09-13T21:46:20.447 INFO:teuthology.orchestra.run.smithi114.stderr:firewalld.service is not enabled 2024-09-13T21:46:20.447 INFO:teuthology.orchestra.run.smithi114.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T21:46:20.447 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for mon to start... 2024-09-13T21:46:20.447 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for mon... 2024-09-13T21:46:20.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:20 smithi114 bash[23364]: 677889c5c4dd50d920301c6e69e69a7d3e0a95882a87c374da5ae71fbe1dccb1 2024-09-13T21:46:20.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:20 smithi114 systemd[1]: Started Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:46:20.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:20 smithi114 ceph-mon[23390]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T21:46:20.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:20 smithi114 ceph-mon[23390]: monmap e1: 1 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0]} 2024-09-13T21:46:20.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:20 smithi114 ceph-mon[23390]: fsmap 2024-09-13T21:46:20.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:20 smithi114 ceph-mon[23390]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T21:46:20.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:20 smithi114 ceph-mon[23390]: mgrmap e1: no daemons active 2024-09-13T21:46:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:23 smithi114 ceph-mon[23390]: from='client.? 172.21.15.114:0/2307850049' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: cluster: 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: id: 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: services: 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 4s) 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: data: 2024-09-13T21:46:24.512 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-13T21:46:24.513 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-13T21:46:24.513 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-13T21:46:24.513 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: pgs: 2024-09-13T21:46:24.513 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:26.820 INFO:teuthology.orchestra.run.smithi114.stderr:mon is available 2024-09-13T21:46:26.820 INFO:teuthology.orchestra.run.smithi114.stderr:Assimilating anything we can from ceph.conf... 2024-09-13T21:46:27.324 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:27.325 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: [global] 2024-09-13T21:46:27.325 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: fsid = 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:27.325 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.114:3300,v1:172.21.15.114:6789] 2024-09-13T21:46:27.328 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T21:46:27.328 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T21:46:27.328 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T21:46:27.328 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T21:46:27.328 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:27.328 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: [mgr] 2024-09-13T21:46:27.329 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = True 2024-09-13T21:46:27.329 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T21:46:27.329 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:27.329 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: [osd] 2024-09-13T21:46:27.329 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T21:46:27.329 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T21:46:27.487 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:27 smithi114 ceph-mon[23390]: from='client.? 172.21.15.114:0/2227592592' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T21:46:27.487 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:27 smithi114 ceph-mon[23390]: from='client.? 172.21.15.114:0/2227592592' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T21:46:27.934 INFO:teuthology.orchestra.run.smithi114.stderr:Generating new minimal ceph.conf... 2024-09-13T21:46:28.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:28 smithi114 ceph-mon[23390]: from='client.? 172.21.15.114:0/3117691536' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:28.966 INFO:teuthology.orchestra.run.smithi114.stderr:Restarting the monitor... 2024-09-13T21:46:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:28 smithi114 systemd[1]: Stopping Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:46:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 bash[23611]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon.a" found: no such container 2024-09-13T21:46:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[23386]: 2024-09-13T21:46:29.123+0000 7f55fcfbd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:46:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[23386]: 2024-09-13T21:46:29.123+0000 7f55fcfbd700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T21:46:29.658 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 podman[23623]: 2024-09-13 21:46:29.379629538 +0000 UTC m=+0.358428605 container died 677889c5c4dd50d920301c6e69e69a7d3e0a95882a87c374da5ae71fbe1dccb1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, version=8) 2024-09-13T21:46:30.000 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 podman[23623]: 2024-09-13 21:46:29.773049082 +0000 UTC m=+0.751848148 container cleanup 677889c5c4dd50d920301c6e69e69a7d3e0a95882a87c374da5ae71fbe1dccb1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, version=8, io.buildah.version=1.19.8, RELEASE=HEAD) 2024-09-13T21:46:30.000 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 bash[23623]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a 2024-09-13T21:46:30.000 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 bash[23650]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon.a" found: no such container 2024-09-13T21:46:30.297 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:29 smithi114 podman[23636]: 2024-09-13 21:46:29.998814442 +0000 UTC m=+0.618836003 container remove 677889c5c4dd50d920301c6e69e69a7d3e0a95882a87c374da5ae71fbe1dccb1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, release=754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-type=git, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, distribution-scope=public, version=8, architecture=x86_64) 2024-09-13T21:46:30.297 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T21:46:30.297 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 systemd[1]: Stopped Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:46:30.297 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 systemd[1]: Starting Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:46:30.297 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 podman[23710]: 2024-09-13 21:46:30.216191101 +0000 UTC m=+0.096454892 container create dae4b66af8adf6d52f1b5df3b3d9813dbda4db14e34e73fb246214ffa6903df3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, release=754, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, name=centos-stream, architecture=x86_64, ceph=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-13T21:46:30.297 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 podman[23710]: 2024-09-13 21:46:30.135944768 +0000 UTC m=+0.016208559 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T21:46:30.472 INFO:teuthology.orchestra.run.smithi114.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-13T21:46:30.692 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 podman[23710]: 2024-09-13 21:46:30.393138916 +0000 UTC m=+0.273402708 container init dae4b66af8adf6d52f1b5df3b3d9813dbda4db14e34e73fb246214ffa6903df3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.openshift.expose-services=, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, ceph=True) 2024-09-13T21:46:30.692 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 podman[23710]: 2024-09-13 21:46:30.396269638 +0000 UTC m=+0.276533430 container start dae4b66af8adf6d52f1b5df3b3d9813dbda4db14e34e73fb246214ffa6903df3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, architecture=x86_64, version=8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T21:46:30.692 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: pidfile_write: ignore empty --pid-file 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: load: jerasure load: lrc 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: RocksDB version: 6.15.5 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Compile date Apr 18 2022 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: DB SUMMARY 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: DB Session ID: LBCVMPZEN1MWX1B076BW 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: CURRENT file: CURRENT 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83459 ; 2024-09-13T21:46:30.693 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.error_if_exists: 0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.create_if_missing: 0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.paranoid_checks: 1 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.env: 0x55eef36c6860 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.fs: Posix File System 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.info_log: 0x55eef4d6ddc0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.statistics: (nil) 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.use_fsync: 0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_log_file_size: 0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T21:46:30.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.allow_fallocate: 1 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.use_direct_reads: 0 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.db_log_dir: 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T21:46:30.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.write_buffer_manager: 0x55eef4e5e240 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T21:46:30.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.unordered_write: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.row_cache: None 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.wal_filter: None 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.preserve_deletes: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.two_write_queues: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.atomic_flush: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T21:46:30.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.log_readahead_size: 0 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_background_jobs: 2 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_background_compactions: -1 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_subcompactions: 1 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T21:46:30.699 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_open_files: -1 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_background_flushes: -1 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Compression algorithms supported: 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kZSTD supported: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kXpressCompression supported: 0 2024-09-13T21:46:30.700 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kLZ4Compression supported: 1 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kBZip2Compression supported: 0 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kZlibCompression supported: 1 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: kSnappyCompression supported: 1 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.merge_operator: 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_filter: None 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T21:46:30.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eef4d39d00) 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: cache_index_and_filter_blocks: 1 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: pin_top_level_index_and_filter: 1 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_type: 0 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: data_block_index_type: 0 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_shortening: 1 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: hash_index_allow_collision: 1 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: checksum: 1 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: no_block_cache: 0 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache: 0x55eef4da4170 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_name: BinnedLRUCache 2024-09-13T21:46:30.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_options: 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: capacity : 536870912 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: num_shard_bits : 4 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: strict_capacity_limit : 0 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: high_pri_pool_ratio: 0.000 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_compressed: (nil) 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: persistent_cache: (nil) 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_size: 4096 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_size_deviation: 10 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_restart_interval: 16 2024-09-13T21:46:30.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_block_restart_interval: 1 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: metadata_block_size: 4096 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: partition_filters: 0 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: use_delta_encoding: 1 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: whole_key_filtering: 1 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: verify_compression: 0 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: read_amp_bytes_per_bit: 0 2024-09-13T21:46:30.704 INFO:journalctl@ceph.mon.a.smithi114.stdout: format_version: 4 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout: enable_index_compression: 1 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_align: 0 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression: NoCompression 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.num_levels: 7 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T21:46:30.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T21:46:30.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T21:46:30.707 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T21:46:30.708 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.table_properties_collectors: 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.inplace_update_support: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.bloom_locality: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.max_successive_merges: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T21:46:30.709 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.ttl: 2592000 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.enable_blob_files: false 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.min_blob_size: 0 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263990422156, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-13T21:46:30.710 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263990422929, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79655, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77899, "index_size": 244, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10073, "raw_average_key_size": 47, "raw_value_size": 72150, "raw_average_value_size": 337, "num_data_blocks": 11, "num_entries": 214, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726263990, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "3b5bcef9-ab59-4baf-9176-b2df0666f9a5", "db_session_id": "LBCVMPZEN1MWX1B076BW"}} 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263990423252, "job": 1, "event": "recovery_finished"} 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55eef4d8aa80 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: DB pointer 0x55eef4d9a000 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** DB Stats ** 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T21:46:30.711 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: L0 2/0 79.55 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: Sum 2/0 79.55 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 143.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:46:30.712 INFO:journalctl@ceph.mon.a.smithi114.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative compaction: 0.00 GB write, 25.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval compaction: 0.00 GB write, 25.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:46:30.713 INFO:journalctl@ceph.mon.a.smithi114.stdout: L0 2/0 79.55 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: Sum 2/0 79.55 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 143.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:46:30.714 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative compaction: 0.00 GB write, 24.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: starting mon.a rank 0 at public addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] at bind addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???) e1 preinit fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???).mds e1 new map 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???).mds e1 print_map 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: e1 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: legacy client fscid: -1 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout: No filesystems configured 2024-09-13T21:46:30.715 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expand map: {default=false} 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta from 'false' to 'false' 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expanded map: {default=false} 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expand map: {default=info} 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta from 'info' to 'info' 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expanded map: {default=info} 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expand map: {default=daemon} 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expanded map: {default=daemon} 2024-09-13T21:46:30.716 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expand map: {default=debug} 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta from 'debug' to 'debug' 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: expand_channel_meta expanded map: {default=debug} 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 bash[23710]: dae4b66af8adf6d52f1b5df3b3d9813dbda4db14e34e73fb246214ffa6903df3 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 systemd[1]: Started Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: monmap e1: 1 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0]} 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: fsmap 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T21:46:30.717 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:30 smithi114 ceph-mon[23730]: mgrmap e1: no daemons active 2024-09-13T21:46:31.520 INFO:teuthology.orchestra.run.smithi114.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-13T21:46:31.521 INFO:teuthology.orchestra.run.smithi114.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:46:31.521 INFO:teuthology.orchestra.run.smithi114.stderr:Creating mgr... 2024-09-13T21:46:31.521 INFO:teuthology.orchestra.run.smithi114.stderr:Verifying port 9283 ... 2024-09-13T21:46:31.752 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Failed to reset failed state of unit ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T21:46:31.759 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: Created symlink /etc/systemd/system/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968.target.wants/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@.service. 2024-09-13T21:46:32.261 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:31 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3148635145' entity='client.admin' 2024-09-13T21:46:32.261 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:32 smithi114 systemd[1]: Starting Ceph mgr.y for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:46:32.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:32 smithi114 podman[23944]: 2024-09-13 21:46:32.162935999 +0000 UTC m=+0.015799336 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T21:46:33.381 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 podman[23944]: 2024-09-13 21:46:33.09073214 +0000 UTC m=+0.943595473 container create f0c0d3e1c0244aea0bc87bd48351d45147159f3693a772b1b0508fa262f065c2 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, version=8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8) 2024-09-13T21:46:33.382 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 podman[23944]: 2024-09-13 21:46:33.30137789 +0000 UTC m=+1.154241221 container init f0c0d3e1c0244aea0bc87bd48351d45147159f3693a772b1b0508fa262f065c2 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, release=754, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc.) 2024-09-13T21:46:33.382 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 podman[23944]: 2024-09-13 21:46:33.30495123 +0000 UTC m=+1.157814560 container start f0c0d3e1c0244aea0bc87bd48351d45147159f3693a772b1b0508fa262f065c2 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, version=8, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git) 2024-09-13T21:46:33.382 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 bash[23944]: f0c0d3e1c0244aea0bc87bd48351d45147159f3693a772b1b0508fa262f065c2 2024-09-13T21:46:33.389 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: disabled 2024-09-13T21:46:33.397 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: inactive 2024-09-13T21:46:33.397 INFO:teuthology.orchestra.run.smithi114.stderr:firewalld.service is not enabled 2024-09-13T21:46:33.398 INFO:teuthology.orchestra.run.smithi114.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T21:46:33.405 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: disabled 2024-09-13T21:46:33.413 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: inactive 2024-09-13T21:46:33.414 INFO:teuthology.orchestra.run.smithi114.stderr:firewalld.service is not enabled 2024-09-13T21:46:33.414 INFO:teuthology.orchestra.run.smithi114.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-13T21:46:33.414 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for mgr to start... 2024-09-13T21:46:33.414 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for mgr... 2024-09-13T21:46:33.664 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 systemd[1]: Started Ceph mgr.y for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:46:33.664 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:33.453+0000 7f3bf70a3000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T21:46:33.664 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:33.531+0000 7f3bf70a3000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T21:46:33.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:33.711+0000 7f3bf70a3000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T21:46:33.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:33.798+0000 7f3bf70a3000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T21:46:33.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:33.932+0000 7f3bf70a3000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:46:34.010 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:34.010 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:46:34.010 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsid": "8ccc9cd8-7219-11ef-bceb-c7b262605968", 2024-09-13T21:46:34.010 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "health": { 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 0 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "a" 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T21:46:34.011 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T21:46:34.012 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T21:46:34.016 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "available": false, 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "iostat", 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "nfs", 2024-09-13T21:46:34.017 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "restful" 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modified": "2024-09-13T21:46:19.540872+0000", 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T21:46:34.018 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:46:34.291 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:34 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1887153469' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T21:46:34.292 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:34.147+0000 7f3bf70a3000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T21:46:34.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:34.486+0000 7f3bf70a3000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T21:46:34.586 INFO:teuthology.orchestra.run.smithi114.stderr:mgr not available, waiting (1/15)... 2024-09-13T21:46:35.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:34.574+0000 7f3bf70a3000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T21:46:35.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:34.738+0000 7f3bf70a3000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T21:46:36.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:35.778+0000 7f3bf70a3000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T21:46:36.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:35.867+0000 7f3bf70a3000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T21:46:36.978 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:36.778+0000 7f3bf70a3000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T21:46:37.087 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsid": "8ccc9cd8-7219-11ef-bceb-c7b262605968", 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "health": { 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 0 2024-09-13T21:46:37.088 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "a" 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T21:46:37.089 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T21:46:37.090 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "available": false, 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "iostat", 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "nfs", 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "restful" 2024-09-13T21:46:37.091 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modified": "2024-09-13T21:46:19.540872+0000", 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T21:46:37.092 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:46:37.229 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1161719891' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T21:46:37.229 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.093+0000 7f3bf70a3000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T21:46:37.229 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.228+0000 7f3bf70a3000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:46:37.505 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.319+0000 7f3bf70a3000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T21:46:37.505 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.417+0000 7f3bf70a3000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T21:46:37.505 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.504+0000 7f3bf70a3000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T21:46:37.693 INFO:teuthology.orchestra.run.smithi114.stderr:mgr not available, waiting (2/15)... 2024-09-13T21:46:37.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.652+0000 7f3bf70a3000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T21:46:37.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.745+0000 7f3bf70a3000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T21:46:38.121 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:37.841+0000 7f3bf70a3000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T21:46:38.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:38.120+0000 7f3bf70a3000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T21:46:38.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:38.206+0000 7f3bf70a3000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T21:46:38.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:38.294+0000 7f3bf70a3000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T21:46:39.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:39.350+0000 7f3bf70a3000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T21:46:39.817 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: Activating manager daemon y 2024-09-13T21:46:39.817 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: mgrmap e2: y(active, starting, since 0.00262175s) 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: Manager daemon y is now available 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' 2024-09-13T21:46:39.818 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:39 smithi114 ceph-mon[23730]: from='mgr.14100 172.21.15.114:0/3352419081' entity='mgr.y' 2024-09-13T21:46:40.190 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:40.190 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsid": "8ccc9cd8-7219-11ef-bceb-c7b262605968", 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "health": { 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 0 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "a" 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-13T21:46:40.191 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T21:46:40.192 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T21:46:40.195 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T21:46:40.195 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:40.196 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "available": false, 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "iostat", 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "nfs", 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "restful" 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T21:46:40.197 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:40.198 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modified": "2024-09-13T21:46:19.540872+0000", 2024-09-13T21:46:40.198 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:40.198 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:40.198 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T21:46:40.198 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:46:40.704 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:40 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/394939527' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T21:46:40.704 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:40 smithi114 ceph-mon[23730]: mgrmap e3: y(active, since 1.00614s) 2024-09-13T21:46:40.706 INFO:teuthology.orchestra.run.smithi114.stderr:mgr not available, waiting (3/15)... 2024-09-13T21:46:42.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:42 smithi114 ceph-mon[23730]: mgrmap e4: y(active, since 2s) 2024-09-13T21:46:43.369 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:43.369 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:46:43.369 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsid": "8ccc9cd8-7219-11ef-bceb-c7b262605968", 2024-09-13T21:46:43.369 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "health": { 2024-09-13T21:46:43.369 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T21:46:43.369 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T21:46:43.369 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 0 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "a" 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T21:46:43.370 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T21:46:43.371 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:43.372 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "available": true, 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "iostat", 2024-09-13T21:46:43.373 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "nfs", 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "restful" 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ], 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "modified": "2024-09-13T21:46:19.540872+0000", 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:46:43.374 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: }, 2024-09-13T21:46:43.375 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T21:46:43.375 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:46:43.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:43 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3548829625' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T21:46:43.979 INFO:teuthology.orchestra.run.smithi114.stderr:mgr is available 2024-09-13T21:46:44.565 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: [global] 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: fsid = 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: [mgr] 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: [osd] 2024-09-13T21:46:44.566 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T21:46:44.567 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T21:46:44.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:44 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3082323509' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T21:46:44.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:44 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3082323509' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T21:46:45.102 INFO:teuthology.orchestra.run.smithi114.stderr:Enabling cephadm module... 2024-09-13T21:46:46.726 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:46 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2015375800' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T21:46:46.726 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setuser ceph since I am not root 2024-09-13T21:46:46.726 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setgroup ceph since I am not root 2024-09-13T21:46:46.726 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:46.595+0000 7fe01dce6000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T21:46:46.726 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:46.675+0000 7fe01dce6000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T21:46:47.047 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:46.847+0000 7fe01dce6000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T21:46:47.047 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:46.935+0000 7fe01dce6000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T21:46:47.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:47.070+0000 7fe01dce6000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:46:47.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:47.283+0000 7fe01dce6000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T21:46:47.625 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:47 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2015375800' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T21:46:47.625 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:47 smithi114 ceph-mon[23730]: mgrmap e5: y(active, since 7s) 2024-09-13T21:46:47.774 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:46:47.775 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-13T21:46:47.775 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "available": true, 2024-09-13T21:46:47.775 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-13T21:46:47.775 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T21:46:47.775 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:46:47.884 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:47.623+0000 7fe01dce6000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T21:46:47.885 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:47.712+0000 7fe01dce6000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T21:46:47.885 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:47.882+0000 7fe01dce6000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T21:46:48.315 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for the mgr to restart... 2024-09-13T21:46:48.315 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for mgr epoch 5... 2024-09-13T21:46:48.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:48 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1265436708' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T21:46:49.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:48.973+0000 7fe01dce6000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T21:46:49.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:49.068+0000 7fe01dce6000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T21:46:50.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:49.984+0000 7fe01dce6000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T21:46:50.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:50.289+0000 7fe01dce6000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T21:46:50.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:50.422+0000 7fe01dce6000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:46:50.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:50.511+0000 7fe01dce6000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T21:46:50.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:50.607+0000 7fe01dce6000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T21:46:50.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:50.693+0000 7fe01dce6000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T21:46:50.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:50.838+0000 7fe01dce6000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T21:46:50.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:50.930+0000 7fe01dce6000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T21:46:51.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:51.026+0000 7fe01dce6000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T21:46:51.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:51.322+0000 7fe01dce6000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T21:46:51.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:51.409+0000 7fe01dce6000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T21:46:51.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:51.498+0000 7fe01dce6000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T21:46:52.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:46:52.556+0000 7fe01dce6000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T21:46:52.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:52 smithi114 ceph-mon[23730]: Active manager daemon y restarted 2024-09-13T21:46:52.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:52 smithi114 ceph-mon[23730]: Activating manager daemon y 2024-09-13T21:46:52.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:52 smithi114 ceph-mon[23730]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T21:46:53.626 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:46:53.627 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-13T21:46:53.627 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T21:46:53.627 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:46:53.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:46:53] ENGINE Bus STARTING 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:46:53] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:46:53] ENGINE Bus STARTED 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: mgrmap e6: y(active, starting, since 0.0525357s) 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: Manager daemon y is now available 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:46:53.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: [13/Sep/2024:21:46:53] ENGINE Bus STARTING 2024-09-13T21:46:53.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:53 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:46:54.137 INFO:teuthology.orchestra.run.smithi114.stderr:mgr epoch 5 is available 2024-09-13T21:46:54.137 INFO:teuthology.orchestra.run.smithi114.stderr:Setting orchestrator backend to cephadm... 2024-09-13T21:46:54.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:54 smithi114 ceph-mon[23730]: [13/Sep/2024:21:46:53] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:46:54.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:54 smithi114 ceph-mon[23730]: [13/Sep/2024:21:46:53] ENGINE Bus STARTED 2024-09-13T21:46:54.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:54 smithi114 ceph-mon[23730]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T21:46:54.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:54 smithi114 ceph-mon[23730]: mgrmap e7: y(active, since 1.05938s) 2024-09-13T21:46:54.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:54 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:54.730 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:54 smithi114 ceph-mon[23730]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T21:46:55.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:55 smithi114 ceph-mon[23730]: mgrmap e8: y(active, since 2s) 2024-09-13T21:46:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:55 smithi114 ceph-mon[23730]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:55 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:46:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:55 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:55.947 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: value unchanged 2024-09-13T21:46:56.507 INFO:teuthology.orchestra.run.smithi114.stderr:Generating ssh key... 2024-09-13T21:46:56.765 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:56 smithi114 ceph-mon[23730]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: Generating public/private rsa key pair. 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: Your identification has been saved in /tmp/tmpj1yk_tv5/key. 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: Your public key has been saved in /tmp/tmpj1yk_tv5/key.pub. 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: The key fingerprint is: 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: SHA256:VA/m+7WNyUfbeY9J+Jng1yvgAOPKY9Txrgob2cm3JiI ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: The key's randomart image is: 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: +---[RSA 3072]----+ 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | + | 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | + o | 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | . . . | 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | = . | 2024-09-13T21:46:57.503 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | o S . . .| 2024-09-13T21:46:57.504 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | +..o o o + *+| 2024-09-13T21:46:57.504 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | +o+... o = *+=| 2024-09-13T21:46:57.504 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | E .+*.... o =.*+| 2024-09-13T21:46:57.504 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: | ..o.=o. ..B.o| 2024-09-13T21:46:57.504 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:46:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: +----[SHA256]-----+ 2024-09-13T21:46:58.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:58 smithi114 ceph-mon[23730]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:58.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:58 smithi114 ceph-mon[23730]: Generating ssh key... 2024-09-13T21:46:58.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:58 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:46:58.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:58 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:46:58.421 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtHfC/sq6ViI4mx7M8DteAOTocGjIWs4ERRr/QCQhwq3m6tEVpFcvEjA5S8jYYYNaC/QL32EJSAqVtQZyHQqRcrh1zwLkXmT2m+SPPwXzW5O8Z1W1QJwQGYLrFuNOk2CzTl0czKT2wqUB9aEoaXEtXNrnI4Hp0KLgfYzHBd4ZhJtb6PCuYwRKqIptn5pWQYFUikEs4eNEJTPUnqKCB2zeJxNn0WU3rrAaVOKYdXVIf3OD7G0V7E8orgxypBqwVEx9wYtmFigCWFI8f9OBn88shyWnfuJFdTiIYW2nMnUOFUssHTummK5HVznXscbkicpKBZi6Ovab22AN9qn+VptHdjWcoYNQgdi24CgAmnE44KRGaDJQ69rYVjRtU9KEtwaBbT1C/PpPV9mF8wp4LAe+Auf4MfgWG6qBRhQLJRzuhiS76dro8741OIMgqNv2A/XNrM3NPxaciuxTtp0VKkXbcopoGyFxic6c9poPUHES/9HBdwOceS5TDX35dybcTARs= ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:46:59.003 INFO:teuthology.orchestra.run.smithi114.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T21:46:59.004 INFO:teuthology.orchestra.run.smithi114.stderr:Adding key to root@localhost authorized_keys... 2024-09-13T21:46:59.009 INFO:teuthology.orchestra.run.smithi114.stderr:Adding host smithi114... 2024-09-13T21:46:59.403 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:46:59 smithi114 ceph-mon[23730]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:00.489 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: Added host 'smithi114' with addr '172.21.15.114' 2024-09-13T21:47:01.069 INFO:teuthology.orchestra.run.smithi114.stderr:Deploying unmanaged mon service... 2024-09-13T21:47:01.779 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:01 smithi114 ceph-mon[23730]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi114", "addr": "172.21.15.114", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:01.779 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:01 smithi114 ceph-mon[23730]: Deploying cephadm binary to smithi114 2024-09-13T21:47:01.779 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:01 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:01.780 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:01 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:01.780 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-13T21:47:02.488 INFO:teuthology.orchestra.run.smithi114.stderr:Deploying unmanaged mgr service... 2024-09-13T21:47:02.757 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:02 smithi114 ceph-mon[23730]: Added host smithi114 2024-09-13T21:47:02.758 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:02 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:03.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:03 smithi114 ceph-mon[23730]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:03.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:03 smithi114 ceph-mon[23730]: Saving service mon spec with placement count:5 2024-09-13T21:47:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:03 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:03.906 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-13T21:47:04.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:04 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:04.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:04 smithi114 ceph-mon[23730]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:04.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:04 smithi114 ceph-mon[23730]: Saving service mgr spec with placement count:2 2024-09-13T21:47:04.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:04 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:06.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:06 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:06.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:06 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:06.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:06 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:47:06.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:06 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' 2024-09-13T21:47:06.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:06 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T21:47:06.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:06 smithi114 ceph-mon[23730]: from='mgr.14120 172.21.15.114:0/2053566817' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]': finished 2024-09-13T21:47:06.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:06 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3012501090' entity='client.admin' 2024-09-13T21:47:07.314 INFO:teuthology.orchestra.run.smithi114.stderr:Enabling the dashboard module... 2024-09-13T21:47:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:07 smithi114 ceph-mon[23730]: Deploying daemon agent.smithi114 on smithi114 2024-09-13T21:47:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:07 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2021460465' entity='client.admin' 2024-09-13T21:47:08.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/175719' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T21:47:08.694 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setuser ceph since I am not root 2024-09-13T21:47:08.694 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setgroup ceph since I am not root 2024-09-13T21:47:08.694 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:08.505+0000 7f8e8c716000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T21:47:08.694 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:08.588+0000 7f8e8c716000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T21:47:08.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:08.763+0000 7f8e8c716000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T21:47:08.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:08.851+0000 7f8e8c716000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T21:47:09.231 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:08.993+0000 7f8e8c716000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:47:09.231 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:09 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:09.206+0000 7f8e8c716000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T21:47:09.549 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:09 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/175719' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T21:47:09.549 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:09 smithi114 ceph-mon[23730]: mgrmap e9: y(active, since 15s) 2024-09-13T21:47:09.673 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:47:09.673 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-13T21:47:09.673 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "available": true, 2024-09-13T21:47:09.673 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-13T21:47:09.673 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T21:47:09.673 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:47:09.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:09 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:09.547+0000 7f8e8c716000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T21:47:09.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:09 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:09.637+0000 7f8e8c716000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T21:47:09.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:09 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:09.807+0000 7f8e8c716000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T21:47:10.269 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for the mgr to restart... 2024-09-13T21:47:10.270 INFO:teuthology.orchestra.run.smithi114.stderr:Waiting for mgr epoch 9... 2024-09-13T21:47:10.390 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:10 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1506131822' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T21:47:10.972 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:10.881+0000 7f8e8c716000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T21:47:10.973 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:10.971+0000 7f8e8c716000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T21:47:12.178 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:11.868+0000 7f8e8c716000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T21:47:12.496 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.177+0000 7f8e8c716000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T21:47:12.496 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.311+0000 7f8e8c716000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:47:12.496 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.399+0000 7f8e8c716000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T21:47:12.496 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.495+0000 7f8e8c716000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T21:47:12.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.583+0000 7f8e8c716000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T21:47:12.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.731+0000 7f8e8c716000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T21:47:13.206 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.824+0000 7f8e8c716000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T21:47:13.206 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:12.921+0000 7f8e8c716000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T21:47:13.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:13.205+0000 7f8e8c716000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T21:47:13.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:13.292+0000 7f8e8c716000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T21:47:13.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:13.381+0000 7f8e8c716000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T21:47:14.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:14 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:47:14.451+0000 7f8e8c716000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T21:47:15.298 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:14 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:47:14] ENGINE Bus STARTING 2024-09-13T21:47:15.298 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:47:15] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:47:15.298 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:47:15] ENGINE Bus STARTED 2024-09-13T21:47:15.518 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: { 2024-09-13T21:47:15.518 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-13T21:47:15.518 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T21:47:15.518 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: } 2024-09-13T21:47:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: Active manager daemon y restarted 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: Activating manager daemon y 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: mgrmap e10: y(active, starting, since 0.0552892s) 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: Manager daemon y is now available 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:15.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:15.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:47:15.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:47:15.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: [13/Sep/2024:21:47:14] ENGINE Bus STARTING 2024-09-13T21:47:15.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: [13/Sep/2024:21:47:15] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:47:15.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: [13/Sep/2024:21:47:15] ENGINE Bus STARTED 2024-09-13T21:47:15.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:16.309 INFO:teuthology.orchestra.run.smithi114.stderr:mgr epoch 9 is available 2024-09-13T21:47:16.309 INFO:teuthology.orchestra.run.smithi114.stderr:Generating a dashboard self-signed certificate... 2024-09-13T21:47:16.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:16 smithi114 ceph-mon[23730]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T21:47:16.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:16 smithi114 ceph-mon[23730]: mgrmap e11: y(active, since 1.05794s) 2024-09-13T21:47:16.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:16 smithi114 ceph-mon[23730]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T21:47:16.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:16.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:47:16.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:16.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T21:47:17.076 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-13T21:47:17.637 INFO:teuthology.orchestra.run.smithi114.stderr:Creating initial admin user... 2024-09-13T21:47:17.885 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:17 smithi114 ceph-mon[23730]: Deploying daemon agent.smithi114 on smithi114 2024-09-13T21:47:17.885 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:17 smithi114 ceph-mon[23730]: mgrmap e12: y(active, since 2s) 2024-09-13T21:47:17.885 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:17 smithi114 ceph-mon[23730]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:17.885 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:17.885 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:18.521 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$uB2lO5rnhbYesPRlkqV.vO4l8AxHyn/hWF5tpI41XZAtm2IIznDfK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726264038, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T21:47:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:18 smithi114 ceph-mon[23730]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:18 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:19.135 INFO:teuthology.orchestra.run.smithi114.stderr:Fetching dashboard port number... 2024-09-13T21:47:19.790 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: 8443 2024-09-13T21:47:20.072 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:19 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1141867367' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T21:47:20.410 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: disabled 2024-09-13T21:47:20.417 INFO:teuthology.orchestra.run.smithi114.stderr:systemctl: inactive 2024-09-13T21:47:20.418 INFO:teuthology.orchestra.run.smithi114.stderr:firewalld.service is not enabled 2024-09-13T21:47:20.418 INFO:teuthology.orchestra.run.smithi114.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T21:47:20.420 INFO:teuthology.orchestra.run.smithi114.stderr:Ceph Dashboard is now available at: 2024-09-13T21:47:20.420 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:20.421 INFO:teuthology.orchestra.run.smithi114.stderr: URL: https://smithi114.front.sepia.ceph.com:8443/ 2024-09-13T21:47:20.421 INFO:teuthology.orchestra.run.smithi114.stderr: User: admin 2024-09-13T21:47:20.421 INFO:teuthology.orchestra.run.smithi114.stderr: Password: 640wpxifmu 2024-09-13T21:47:20.421 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:20.421 INFO:teuthology.orchestra.run.smithi114.stderr:Enabling autotune for osd_memory_target 2024-09-13T21:47:22.366 INFO:teuthology.orchestra.run.smithi114.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-13T21:47:22.751 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:22 smithi114 ceph-mon[23730]: mgrmap e13: y(active, since 6s) 2024-09-13T21:47:22.751 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:22 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3036378761' entity='client.admin' 2024-09-13T21:47:23.023 INFO:teuthology.orchestra.run.smithi114.stderr:You can access the Ceph CLI with: 2024-09-13T21:47:23.023 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:23.023 INFO:teuthology.orchestra.run.smithi114.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:47:23.023 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:23.023 INFO:teuthology.orchestra.run.smithi114.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-13T21:47:23.023 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:23.024 INFO:teuthology.orchestra.run.smithi114.stderr: ceph telemetry on 2024-09-13T21:47:23.024 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:23.024 INFO:teuthology.orchestra.run.smithi114.stderr:For more information see: 2024-09-13T21:47:23.024 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:23.024 INFO:teuthology.orchestra.run.smithi114.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T21:47:23.024 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T21:47:23.024 INFO:teuthology.orchestra.run.smithi114.stderr:Bootstrap complete. 2024-09-13T21:47:23.056 INFO:tasks.cephadm:Fetching config... 2024-09-13T21:47:23.057 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:47:23.057 DEBUG:teuthology.orchestra.run.smithi114:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T21:47:23.073 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T21:47:23.073 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:47:23.073 DEBUG:teuthology.orchestra.run.smithi114:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T21:47:23.132 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T21:47:23.133 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:47:23.133 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T21:47:23.242 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T21:47:23.243 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:47:23.243 DEBUG:teuthology.orchestra.run.smithi114:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T21:47:23.298 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T21:47:23.298 DEBUG:teuthology.orchestra.run.smithi114:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtHfC/sq6ViI4mx7M8DteAOTocGjIWs4ERRr/QCQhwq3m6tEVpFcvEjA5S8jYYYNaC/QL32EJSAqVtQZyHQqRcrh1zwLkXmT2m+SPPwXzW5O8Z1W1QJwQGYLrFuNOk2CzTl0czKT2wqUB9aEoaXEtXNrnI4Hp0KLgfYzHBd4ZhJtb6PCuYwRKqIptn5pWQYFUikEs4eNEJTPUnqKCB2zeJxNn0WU3rrAaVOKYdXVIf3OD7G0V7E8orgxypBqwVEx9wYtmFigCWFI8f9OBn88shyWnfuJFdTiIYW2nMnUOFUssHTummK5HVznXscbkicpKBZi6Ovab22AN9qn+VptHdjWcoYNQgdi24CgAmnE44KRGaDJQ69rYVjRtU9KEtwaBbT1C/PpPV9mF8wp4LAe+Auf4MfgWG6qBRhQLJRzuhiS76dro8741OIMgqNv2A/XNrM3NPxaciuxTtp0VKkXbcopoGyFxic6c9poPUHES/9HBdwOceS5TDX35dybcTARs= ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T21:47:23.440 INFO:teuthology.orchestra.run.smithi114.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtHfC/sq6ViI4mx7M8DteAOTocGjIWs4ERRr/QCQhwq3m6tEVpFcvEjA5S8jYYYNaC/QL32EJSAqVtQZyHQqRcrh1zwLkXmT2m+SPPwXzW5O8Z1W1QJwQGYLrFuNOk2CzTl0czKT2wqUB9aEoaXEtXNrnI4Hp0KLgfYzHBd4ZhJtb6PCuYwRKqIptn5pWQYFUikEs4eNEJTPUnqKCB2zeJxNn0WU3rrAaVOKYdXVIf3OD7G0V7E8orgxypBqwVEx9wYtmFigCWFI8f9OBn88shyWnfuJFdTiIYW2nMnUOFUssHTummK5HVznXscbkicpKBZi6Ovab22AN9qn+VptHdjWcoYNQgdi24CgAmnE44KRGaDJQ69rYVjRtU9KEtwaBbT1C/PpPV9mF8wp4LAe+Auf4MfgWG6qBRhQLJRzuhiS76dro8741OIMgqNv2A/XNrM3NPxaciuxTtp0VKkXbcopoGyFxic6c9poPUHES/9HBdwOceS5TDX35dybcTARs= ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:47:23.453 DEBUG:teuthology.orchestra.run.smithi121:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtHfC/sq6ViI4mx7M8DteAOTocGjIWs4ERRr/QCQhwq3m6tEVpFcvEjA5S8jYYYNaC/QL32EJSAqVtQZyHQqRcrh1zwLkXmT2m+SPPwXzW5O8Z1W1QJwQGYLrFuNOk2CzTl0czKT2wqUB9aEoaXEtXNrnI4Hp0KLgfYzHBd4ZhJtb6PCuYwRKqIptn5pWQYFUikEs4eNEJTPUnqKCB2zeJxNn0WU3rrAaVOKYdXVIf3OD7G0V7E8orgxypBqwVEx9wYtmFigCWFI8f9OBn88shyWnfuJFdTiIYW2nMnUOFUssHTummK5HVznXscbkicpKBZi6Ovab22AN9qn+VptHdjWcoYNQgdi24CgAmnE44KRGaDJQ69rYVjRtU9KEtwaBbT1C/PpPV9mF8wp4LAe+Auf4MfgWG6qBRhQLJRzuhiS76dro8741OIMgqNv2A/XNrM3NPxaciuxTtp0VKkXbcopoGyFxic6c9poPUHES/9HBdwOceS5TDX35dybcTARs= ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T21:47:23.557 INFO:teuthology.orchestra.run.smithi121.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtHfC/sq6ViI4mx7M8DteAOTocGjIWs4ERRr/QCQhwq3m6tEVpFcvEjA5S8jYYYNaC/QL32EJSAqVtQZyHQqRcrh1zwLkXmT2m+SPPwXzW5O8Z1W1QJwQGYLrFuNOk2CzTl0czKT2wqUB9aEoaXEtXNrnI4Hp0KLgfYzHBd4ZhJtb6PCuYwRKqIptn5pWQYFUikEs4eNEJTPUnqKCB2zeJxNn0WU3rrAaVOKYdXVIf3OD7G0V7E8orgxypBqwVEx9wYtmFigCWFI8f9OBn88shyWnfuJFdTiIYW2nMnUOFUssHTummK5HVznXscbkicpKBZi6Ovab22AN9qn+VptHdjWcoYNQgdi24CgAmnE44KRGaDJQ69rYVjRtU9KEtwaBbT1C/PpPV9mF8wp4LAe+Auf4MfgWG6qBRhQLJRzuhiS76dro8741OIMgqNv2A/XNrM3NPxaciuxTtp0VKkXbcopoGyFxic6c9poPUHES/9HBdwOceS5TDX35dybcTARs= ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:47:23.571 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T21:47:24.472 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:47:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:24.811 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:24.811 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:24.811 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:24.812 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:24.812 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:24.812 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:25.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:25 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:27.188 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T21:47:27.189 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T21:47:27.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:27 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3447363578' entity='client.admin' 2024-09-13T21:47:27.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:27 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:27.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:27 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:30.441 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:47:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:30.740 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi121 2024-09-13T21:47:30.741 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-09-13T21:47:30.741 DEBUG:teuthology.orchestra.run.smithi121:> dd of=/etc/ceph/ceph.conf 2024-09-13T21:47:30.760 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-09-13T21:47:30.760 DEBUG:teuthology.orchestra.run.smithi121:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:47:30.818 INFO:tasks.cephadm:Adding host smithi121 to orchestrator... 2024-09-13T21:47:30.818 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch host add smithi121 2024-09-13T21:47:31.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:47:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:30 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:32.183 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:32.184 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:33.525 INFO:teuthology.orchestra.run.smithi114.stdout:Added host 'smithi121' with addr '172.21.15.121' 2024-09-13T21:47:33.771 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:33 smithi114 ceph-mon[23730]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi121", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:33.771 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:33 smithi114 ceph-mon[23730]: Deploying cephadm binary to smithi121 2024-09-13T21:47:34.080 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T21:47:34.636 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:34.636 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: Added host smithi121 2024-09-13T21:47:34.636 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]': finished 2024-09-13T21:47:34.637 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:34 smithi114 ceph-mon[23730]: Deploying daemon agent.smithi121 on smithi121 2024-09-13T21:47:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:35.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:35.897 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:47:35.897 INFO:teuthology.orchestra.run.smithi114.stdout:[{"addr": "172.21.15.114", "hostname": "smithi114", "labels": [], "status": ""}, {"addr": "172.21.15.121", "hostname": "smithi121", "labels": [], "status": ""}] 2024-09-13T21:47:36.192 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:47:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:36.468 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:47:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:36.496 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T21:47:36.496 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T21:47:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:36 smithi114 ceph-mon[23730]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:47:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:37.797 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:47:37] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:37 smithi114 ceph-mon[23730]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:38.877 INFO:teuthology.orchestra.run.smithi114.stderr:adjusted tunables profile to default 2024-09-13T21:47:39.180 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:38 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/32039115' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T21:47:39.468 INFO:tasks.cephadm:Adding mon.a on smithi114 2024-09-13T21:47:39.468 INFO:tasks.cephadm:Adding mon.c on smithi114 2024-09-13T21:47:39.468 INFO:tasks.cephadm:Adding mon.b on smithi121 2024-09-13T21:47:39.468 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi114:172.21.15.114=a;smithi114:[v2:172.21.15.114:3301,v1:172.21.15.114:6790]=c;smithi121:172.21.15.121=b' 2024-09-13T21:47:40.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:39 smithi114 ceph-mon[23730]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:39 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/32039115' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T21:47:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:39 smithi114 ceph-mon[23730]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:47:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:39 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:41.337 INFO:teuthology.orchestra.run.smithi121.stdout:Scheduled mon update... 2024-09-13T21:47:42.171 DEBUG:teuthology.orchestra.run.smithi114:mon.c> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T21:47:42.174 DEBUG:teuthology.orchestra.run.smithi121:mon.b> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T21:47:42.176 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T21:47:42.176 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:47:42.687 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:42.687 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi114:172.21.15.114=a;smithi114:[v2:172.21.15.114:3301,v1:172.21.15.114:6790]=c;smithi121:172.21.15.121=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:42.687 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: Saving service mon spec with placement smithi114:172.21.15.114=a;smithi114:[v2:172.21.15.114:3301,v1:172.21.15.114:6790]=c;smithi121:172.21.15.121=b;count:3 2024-09-13T21:47:42.687 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:42.688 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:42.688 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:42.688 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:42.688 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:42.688 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:47:42.688 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:42.688 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:42 smithi114 ceph-mon[23730]: Deploying daemon mon.c on smithi114 2024-09-13T21:47:44.122 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:47:44.122 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":1,"fsid":"8ccc9cd8-7219-11ef-bceb-c7b262605968","modified":"2024-09-13T21:46:14.012833Z","created":"2024-09-13T21:46:14.012833Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3300","nonce":0},{"type":"v1","addr":"172.21.15.114:6789","nonce":0}]},"addr":"172.21.15.114:6789/0","public_addr":"172.21.15.114:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:47:44.124 INFO:teuthology.orchestra.run.smithi121.stderr:dumped monmap epoch 1 2024-09-13T21:47:44.227 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 systemd[1]: Starting Ceph mon.c for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:47:44.487 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[23730]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:44.488 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/2191022173' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:47:44.488 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 podman[28509]: 2024-09-13 21:47:44.12970839 +0000 UTC m=+0.016390801 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T21:47:44.488 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 podman[28509]: 2024-09-13 21:47:44.244564424 +0000 UTC m=+0.131246865 container create 92fe3168759696038ddc4da7e23df6e10f4d90b8120487116bbe6949076eac74 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, architecture=x86_64, version=8) 2024-09-13T21:47:44.488 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 podman[28509]: 2024-09-13 21:47:44.463792175 +0000 UTC m=+0.350474583 container init 92fe3168759696038ddc4da7e23df6e10f4d90b8120487116bbe6949076eac74 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_CLEAN=True, RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, release=754, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8) 2024-09-13T21:47:44.488 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 podman[28509]: 2024-09-13 21:47:44.467147344 +0000 UTC m=+0.353829743 container start 92fe3168759696038ddc4da7e23df6e10f4d90b8120487116bbe6949076eac74 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, distribution-scope=public, ceph=True, name=centos-stream, version=8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8) 2024-09-13T21:47:44.488 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T21:47:44.489 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T21:47:44.489 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: pidfile_write: ignore empty --pid-file 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: load: jerasure load: lrc 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: RocksDB version: 6.15.5 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Compile date Apr 18 2022 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: DB SUMMARY 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: DB Session ID: TRJZQ98NPW1CYTJJXFNP 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: CURRENT file: CURRENT 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.error_if_exists: 0 2024-09-13T21:47:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.create_if_missing: 0 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.paranoid_checks: 1 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.env: 0x5562190c6860 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.fs: Posix File System 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.info_log: 0x55621a8f5dc0 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.statistics: (nil) 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.use_fsync: 0 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_log_file_size: 0 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T21:47:44.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.allow_fallocate: 1 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.use_direct_reads: 0 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.db_log_dir: 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T21:47:44.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.write_buffer_manager: 0x55621a9e6270 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T21:47:44.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.unordered_write: 0 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.row_cache: None 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.wal_filter: None 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.preserve_deletes: 0 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.two_write_queues: 0 2024-09-13T21:47:44.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.atomic_flush: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.log_readahead_size: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_background_jobs: 2 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_background_compactions: -1 2024-09-13T21:47:44.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_subcompactions: 1 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_open_files: -1 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T21:47:44.832 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_background_flushes: -1 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Compression algorithms supported: 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kZSTD supported: 0 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kXpressCompression supported: 0 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kLZ4Compression supported: 1 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kBZip2Compression supported: 0 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kZlibCompression supported: 1 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: kSnappyCompression supported: 1 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.merge_operator: 2024-09-13T21:47:44.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_filter: None 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55621a8c1d00) 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: cache_index_and_filter_blocks: 1 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: pin_top_level_index_and_filter: 1 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: index_type: 0 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: data_block_index_type: 0 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: index_shortening: 1 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T21:47:44.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: hash_index_allow_collision: 1 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: checksum: 1 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: no_block_cache: 0 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache: 0x55621a92c170 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache_name: BinnedLRUCache 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache_options: 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: capacity : 536870912 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: num_shard_bits : 4 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: strict_capacity_limit : 0 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: high_pri_pool_ratio: 0.000 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache_compressed: (nil) 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: persistent_cache: (nil) 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_size: 4096 2024-09-13T21:47:44.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_size_deviation: 10 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_restart_interval: 16 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: index_block_restart_interval: 1 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: metadata_block_size: 4096 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: partition_filters: 0 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: use_delta_encoding: 1 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: whole_key_filtering: 1 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: verify_compression: 0 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: read_amp_bytes_per_bit: 0 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: format_version: 4 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: enable_index_compression: 1 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_align: 0 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T21:47:44.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression: NoCompression 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.num_levels: 7 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T21:47:44.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T21:47:44.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T21:47:44.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.table_properties_collectors: 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.inplace_update_support: 0 2024-09-13T21:47:44.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.bloom_locality: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.max_successive_merges: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.ttl: 2592000 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.enable_blob_files: false 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.min_blob_size: 0 2024-09-13T21:47:44.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726264064493352, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726264064493639, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726264064, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "fb947ab7-7802-451b-a8df-078f9aaada4c", "db_session_id": "TRJZQ98NPW1CYTJJXFNP"}} 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726264064493933, "job": 1, "event": "recovery_finished"} 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55621a912a80 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: DB pointer 0x55621a922000 2024-09-13T21:47:44.842 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** DB Stats ** 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T21:47:44.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:47:44.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T21:47:44.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: using public_addrv [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: starting mon.c rank -1 at public addrs [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] at bind addrs [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(???) e0 preinit fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:47:44.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).mds e1 new map 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout: e1 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout: legacy client fscid: -1 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout: No filesystems configured 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T21:47:44.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/32039115' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi114:172.21.15.114=a;smithi114:[v2:172.21.15.114:3301,v1:172.21.15.114:6790]=c;smithi121:172.21.15.121=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: Saving service mon spec with placement smithi114:172.21.15.114=a;smithi114:[v2:172.21.15.114:3301,v1:172.21.15.114:6790]=c;smithi121:172.21.15.121=b;count:3 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:44.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: Deploying daemon mon.c on smithi114 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/2191022173' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing).paxosservice(auth 1..5) refresh upgraded, format 0 -> 3 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 ceph-mon[28528]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 bash[28509]: 92fe3168759696038ddc4da7e23df6e10f4d90b8120487116bbe6949076eac74 2024-09-13T21:47:44.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:44 smithi114 systemd[1]: Started Ceph mon.c for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:47:45.706 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T21:47:45.706 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:47:49.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:47:49] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: mon.a calling monitor election 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: mon.c calling monitor election 2024-09-13T21:47:49.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: monmap e2: 2 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: fsmap 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: mgrmap e13: y(active, since 35s) 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: overall HEALTH_OK 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:49.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: mon.a calling monitor election 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: mon.c calling monitor election 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:49.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: monmap e2: 2 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: fsmap 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: mgrmap e13: y(active, since 35s) 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: overall HEALTH_OK 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:49.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:49 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:50.487 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:47:50.487 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":2,"fsid":"8ccc9cd8-7219-11ef-bceb-c7b262605968","modified":"2024-09-13T21:47:44.513230Z","created":"2024-09-13T21:46:14.012833Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3300","nonce":0},{"type":"v1","addr":"172.21.15.114:6789","nonce":0}]},"addr":"172.21.15.114:6789/0","public_addr":"172.21.15.114:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3301","nonce":0},{"type":"v1","addr":"172.21.15.114:6790","nonce":0}]},"addr":"172.21.15.114:6790/0","public_addr":"172.21.15.114:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T21:47:50.489 INFO:teuthology.orchestra.run.smithi121.stderr:dumped monmap epoch 2 2024-09-13T21:47:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:50 smithi114 ceph-mon[23730]: Deploying daemon mon.b on smithi121 2024-09-13T21:47:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:50 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/1290573572' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:47:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:50 smithi114 ceph-mon[28528]: Deploying daemon mon.b on smithi121 2024-09-13T21:47:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:50 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/1290573572' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:47:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:51 smithi114 ceph-mon[23730]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:51 smithi114 ceph-mon[28528]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:52.282 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T21:47:52.282 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:47:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:53 smithi114 ceph-mon[23730]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:53 smithi114 ceph-mon[28528]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:54.309 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:47:54.309 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":2,"fsid":"8ccc9cd8-7219-11ef-bceb-c7b262605968","modified":"2024-09-13T21:47:44.513230Z","created":"2024-09-13T21:46:14.012833Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3300","nonce":0},{"type":"v1","addr":"172.21.15.114:6789","nonce":0}]},"addr":"172.21.15.114:6789/0","public_addr":"172.21.15.114:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3301","nonce":0},{"type":"v1","addr":"172.21.15.114:6790","nonce":0}]},"addr":"172.21.15.114:6790/0","public_addr":"172.21.15.114:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T21:47:54.310 INFO:teuthology.orchestra.run.smithi121.stderr:dumped monmap epoch 2 2024-09-13T21:47:56.011 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T21:47:56.011 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:47:59.695 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:47:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:47:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: mon.a calling monitor election 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: mon.c calling monitor election 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:59.696 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: fsmap 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: mgrmap e13: y(active, since 44s) 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-13T21:47:59.697 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: mon.b (rank 2) addr [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] is down (out of quorum) 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: mon.a calling monitor election 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: mon.c calling monitor election 2024-09-13T21:47:59.698 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: fsmap 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: mgrmap e13: y(active, since 44s) 2024-09-13T21:47:59.699 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: mon.b (rank 2) addr [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] is down (out of quorum) 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:47:59.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:47:59 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:47:59.945 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":3,"fsid":"8ccc9cd8-7219-11ef-bceb-c7b262605968","modified":"2024-09-13T21:47:54.401708Z","created":"2024-09-13T21:46:14.012833Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3300","nonce":0},{"type":"v1","addr":"172.21.15.114:6789","nonce":0}]},"addr":"172.21.15.114:6789/0","public_addr":"172.21.15.114:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3301","nonce":0},{"type":"v1","addr":"172.21.15.114:6790","nonce":0}]},"addr":"172.21.15.114:6790/0","public_addr":"172.21.15.114:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T21:47:59.945 INFO:teuthology.orchestra.run.smithi121.stderr:dumped monmap epoch 3 2024-09-13T21:48:00.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:48:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: Reconfiguring daemon mon.a on smithi114 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/2940368692' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: Reconfiguring daemon mon.a on smithi114 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/2940368692' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:48:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:48:00.918 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T21:48:00.918 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T21:48:01.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:48:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:02.262 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: mon.b calling monitor election 2024-09-13T21:48:02.262 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: mon.b calling monitor election 2024-09-13T21:48:02.262 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: mon.c calling monitor election 2024-09-13T21:48:02.262 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: mon.a calling monitor election 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: fsmap 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: mgrmap e13: y(active, since 46s) 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: overall HEALTH_OK 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:02.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:02.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: mon.b calling monitor election 2024-09-13T21:48:02.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: mon.b calling monitor election 2024-09-13T21:48:02.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: mon.c calling monitor election 2024-09-13T21:48:02.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: mon.a calling monitor election 2024-09-13T21:48:02.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T21:48:02.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T21:48:02.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: fsmap 2024-09-13T21:48:02.265 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:48:02.265 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: mgrmap e13: y(active, since 46s) 2024-09-13T21:48:02.265 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T21:48:02.265 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T21:48:02.265 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: overall HEALTH_OK 2024-09-13T21:48:02.265 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:02.265 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:03.020 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:02 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:48:03.177 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:02 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:48:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:02 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:48:03.177 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:48:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:03.576 INFO:teuthology.orchestra.run.smithi114.stdout:# minimal ceph.conf for 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:48:03.576 INFO:teuthology.orchestra.run.smithi114.stdout:[global] 2024-09-13T21:48:03.576 INFO:teuthology.orchestra.run.smithi114.stdout: fsid = 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:48:03.576 INFO:teuthology.orchestra.run.smithi114.stdout: mon_host = [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] 2024-09-13T21:48:04.239 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T21:48:04.239 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:48:04.239 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T21:48:04.265 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[23730]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:04.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:04.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:04.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1731370' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:04.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[28528]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:04.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:04.267 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:04.267 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:04 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1731370' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:04.267 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:48:04.268 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:48:04.332 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-09-13T21:48:04.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T21:48:04.362 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:04 smithi121 ceph-mon[24681]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:04.362 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:04 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:04.362 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:04 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:04.362 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:04 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1731370' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:04.368 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-09-13T21:48:04.368 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:48:04.437 INFO:tasks.cephadm:Adding mgr.y on smithi114 2024-09-13T21:48:04.438 INFO:tasks.cephadm:Adding mgr.x on smithi121 2024-09-13T21:48:04.438 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi114=y;smithi121=x' 2024-09-13T21:48:06.206 INFO:teuthology.orchestra.run.smithi121.stdout:Scheduled mgr update... 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: Reconfiguring mon.b (monmap changed)... 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: Reconfiguring daemon mon.b on smithi121 2024-09-13T21:48:06.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: Reconfiguring mon.b (monmap changed)... 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: Reconfiguring daemon mon.b on smithi121 2024-09-13T21:48:06.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: Reconfiguring mon.b (monmap changed)... 2024-09-13T21:48:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:48:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:48:06.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:06.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: Reconfiguring daemon mon.b on smithi121 2024-09-13T21:48:06.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:06.797 DEBUG:teuthology.orchestra.run.smithi121:mgr.x> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T21:48:06.799 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T21:48:06.800 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:48:06.800 DEBUG:teuthology.orchestra.run.smithi114:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T21:48:06.817 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T21:48:06.817 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_1 2024-09-13T21:48:06.873 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T21:48:06.873 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:06.873 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-13T21:48:06.873 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:06.874 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:06.874 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-09-13 21:48:06.242020242 +0000 2024-09-13T21:48:06.874 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-09-13 21:45:31.487253113 +0000 2024-09-13T21:48:06.874 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-09-13 21:45:31.487253113 +0000 2024-09-13T21:48:06.874 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-09-13 21:45:31.487253113 +0000 2024-09-13T21:48:06.874 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T21:48:06.939 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-09-13T21:48:06.939 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-09-13T21:48:06.939 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000246896 s, 2.1 MB/s 2024-09-13T21:48:06.940 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T21:48:06.998 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_2 2024-09-13T21:48:07.054 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T21:48:07.054 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:07.055 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-13T21:48:07.055 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:07.055 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:07.055 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-09-13 21:48:06.242020242 +0000 2024-09-13T21:48:07.055 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-09-13 21:45:31.809245869 +0000 2024-09-13T21:48:07.055 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-09-13 21:45:31.809245869 +0000 2024-09-13T21:48:07.055 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-09-13 21:45:31.809245869 +0000 2024-09-13T21:48:07.055 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T21:48:07.118 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-09-13T21:48:07.118 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-09-13T21:48:07.118 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000288957 s, 1.8 MB/s 2024-09-13T21:48:07.119 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T21:48:07.176 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_3 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-09-13 21:48:06.242020242 +0000 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-09-13 21:45:32.122238828 +0000 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-09-13 21:45:32.122238828 +0000 2024-09-13T21:48:07.231 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-09-13 21:45:32.122238828 +0000 2024-09-13T21:48:07.232 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T21:48:07.238 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:48:07] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:07.261 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-09-13T21:48:07.261 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-09-13T21:48:07.261 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000241088 s, 2.1 MB/s 2024-09-13T21:48:07.262 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T21:48:07.318 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_4 2024-09-13T21:48:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:07 smithi121 ceph-mon[24681]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi114=y;smithi121=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:07 smithi121 ceph-mon[24681]: Saving service mgr spec with placement smithi114=y;smithi121=x;count:2 2024-09-13T21:48:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:07 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:07 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-09-13 21:48:06.242020242 +0000 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-09-13 21:45:32.443231607 +0000 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-09-13 21:45:32.443231607 +0000 2024-09-13T21:48:07.374 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-09-13 21:45:32.443231607 +0000 2024-09-13T21:48:07.375 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T21:48:07.437 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-09-13T21:48:07.437 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-09-13T21:48:07.438 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000237899 s, 2.2 MB/s 2024-09-13T21:48:07.439 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T21:48:07.496 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-09-13T21:48:07.496 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T21:48:07.512 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T21:48:07.512 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_1 2024-09-13T21:48:07.568 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T21:48:07.568 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:07.568 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-13T21:48:07.569 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:07.569 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:07.569 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-09-13 21:48:02.270803983 +0000 2024-09-13T21:48:07.569 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-09-13 21:45:30.539069929 +0000 2024-09-13T21:48:07.569 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-09-13 21:45:30.539069929 +0000 2024-09-13T21:48:07.569 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-09-13 21:45:30.539069929 +0000 2024-09-13T21:48:07.569 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T21:48:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[23730]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi114=y;smithi121=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[23730]: Saving service mgr spec with placement smithi114=y;smithi121=x;count:2 2024-09-13T21:48:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:07.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[28528]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi114=y;smithi121=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:07.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[28528]: Saving service mgr spec with placement smithi114=y;smithi121=x;count:2 2024-09-13T21:48:07.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:07.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:07 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:07.631 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-09-13T21:48:07.631 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-09-13T21:48:07.632 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000212797 s, 2.4 MB/s 2024-09-13T21:48:07.633 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T21:48:07.689 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_2 2024-09-13T21:48:07.745 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T21:48:07.745 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:07.745 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-13T21:48:07.745 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:07.745 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:07.745 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-09-13 21:48:02.270803983 +0000 2024-09-13T21:48:07.745 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-09-13 21:45:30.866065503 +0000 2024-09-13T21:48:07.746 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-09-13 21:45:30.866065503 +0000 2024-09-13T21:48:07.746 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-09-13 21:45:30.866065503 +0000 2024-09-13T21:48:07.746 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T21:48:07.810 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-09-13T21:48:07.810 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-09-13T21:48:07.810 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000214748 s, 2.4 MB/s 2024-09-13T21:48:07.811 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T21:48:07.867 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_3 2024-09-13T21:48:07.922 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-09-13 21:48:02.270803983 +0000 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-09-13 21:45:31.196061037 +0000 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-09-13 21:45:31.196061037 +0000 2024-09-13T21:48:07.923 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-09-13 21:45:31.196061037 +0000 2024-09-13T21:48:07.923 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T21:48:07.986 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-09-13T21:48:07.987 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-09-13T21:48:07.987 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000217717 s, 2.4 MB/s 2024-09-13T21:48:07.988 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T21:48:08.044 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_4 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-09-13 21:48:02.270803983 +0000 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-09-13 21:45:31.528056544 +0000 2024-09-13T21:48:08.100 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-09-13 21:45:31.528056544 +0000 2024-09-13T21:48:08.101 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: 2024-09-13 21:45:31.528056544 +0000 2024-09-13T21:48:08.101 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T21:48:08.163 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-09-13T21:48:08.164 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-09-13T21:48:08.164 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000202299 s, 2.5 MB/s 2024-09-13T21:48:08.165 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T21:48:08.221 INFO:tasks.cephadm:Deploying osd.0 on smithi114 with /dev/vg_nvme/lv_4... 2024-09-13T21:48:08.221 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T21:48:08.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:08.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:08.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:08.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:08.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:08.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:08.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:48:08.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T21:48:08.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:48:08.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:08.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:08.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:08.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:08.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:08.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:48:08.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T21:48:08.528 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:48:08.528 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:09.508 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:09 smithi114 ceph-mon[23730]: Deploying daemon mgr.x on smithi121 2024-09-13T21:48:09.508 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:09 smithi114 ceph-mon[28528]: Deploying daemon mgr.x on smithi121 2024-09-13T21:48:09.510 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:09 smithi121 ceph-mon[24681]: Deploying daemon mgr.x on smithi121 2024-09-13T21:48:09.510 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 systemd[1]: Starting Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:48:09.831 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 podman[26194]: 2024-09-13 21:48:09.412317472 +0000 UTC m=+0.016187582 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T21:48:09.831 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 podman[26194]: 2024-09-13 21:48:09.511308761 +0000 UTC m=+0.115178874 container create 7f14340b5cbc33b2393bc01600a2adbb3ce4720d1386ef59b0b93c39e70a1dd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, io.k8s.display-name=CentOS Stream 8, vcs-type=git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, name=centos-stream, io.openshift.tags=base centos centos-stream, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_BRANCH=HEAD) 2024-09-13T21:48:09.832 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 podman[26194]: 2024-09-13 21:48:09.730786315 +0000 UTC m=+0.334656434 container init 7f14340b5cbc33b2393bc01600a2adbb3ce4720d1386ef59b0b93c39e70a1dd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, maintainer=Guillaume Abrioux , version=8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_CLEAN=True, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, architecture=x86_64, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8) 2024-09-13T21:48:09.832 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 podman[26194]: 2024-09-13 21:48:09.733976529 +0000 UTC m=+0.337846640 container start 7f14340b5cbc33b2393bc01600a2adbb3ce4720d1386ef59b0b93c39e70a1dd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, version=8, vendor=Red Hat, Inc., release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T21:48:09.832 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 ceph-mgr[26213]: -- 172.21.15.121:0/242766517 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e5bf7d0340 con 0x55e5bf7a8c00 2024-09-13T21:48:09.832 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 bash[26194]: 7f14340b5cbc33b2393bc01600a2adbb3ce4720d1386ef59b0b93c39e70a1dd3 2024-09-13T21:48:10.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 systemd[1]: Started Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:48:10.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:09.877+0000 7f4322789000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T21:48:10.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:09.957+0000 7f4322789000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T21:48:10.392 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:10.130+0000 7f4322789000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:10.229+0000 7f4322789000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:10.361+0000 7f4322789000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:10.393 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:10.394 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:48:10.394 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:48:10.394 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:10.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:10.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:10.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:10.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:10.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:10.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:48:10.534 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:48:10.535 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:10 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:10.604 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:48:10.622 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_4 2024-09-13T21:48:10.786 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:10.578+0000 7f4322789000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T21:48:11.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:10.918+0000 7f4322789000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T21:48:11.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:11.008+0000 7f4322789000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T21:48:11.461 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:11.174+0000 7f4322789000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T21:48:11.461 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:11 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:11.462 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:11 smithi121 ceph-mon[24681]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T21:48:11.462 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:11 smithi121 ceph-mon[24681]: Reconfiguring daemon mgr.y on smithi114 2024-09-13T21:48:11.532 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:11 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:11.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:11 smithi114 ceph-mon[28528]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T21:48:11.533 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:11 smithi114 ceph-mon[28528]: Reconfiguring daemon mgr.y on smithi114 2024-09-13T21:48:11.533 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:11 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:11.533 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:11 smithi114 ceph-mon[23730]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T21:48:11.533 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:11 smithi114 ceph-mon[23730]: Reconfiguring daemon mgr.y on smithi114 2024-09-13T21:48:12.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[28528]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[23730]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:12.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-mon[24681]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:12.344+0000 7f4322789000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T21:48:12.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:12 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:12.440+0000 7f4322789000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T21:48:13.355 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:13 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:13.355 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:13 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:13.355 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:48:13] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:13.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:13 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:48:13.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:13.375+0000 7f4322789000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T21:48:13.632 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:48:13] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:13.991 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:13.681+0000 7f4322789000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T21:48:13.992 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:13.810+0000 7f4322789000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:48:13.992 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:13.895+0000 7f4322789000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T21:48:13.992 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:13.990+0000 7f4322789000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T21:48:14.253 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:14.075+0000 7f4322789000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T21:48:14.253 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:14.218+0000 7f4322789000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.507 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.507 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:14.507 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:14.507 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:14.507 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:14.310+0000 7f4322789000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T21:48:14.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:14.404+0000 7f4322789000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:15.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:14.684+0000 7f4322789000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T21:48:15.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:14.770+0000 7f4322789000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T21:48:15.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:14.856+0000 7f4322789000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T21:48:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:48:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:48:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:15.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:48:15.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:48:15.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:48:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:48:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:16.257 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:48:15 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:48:15.903+0000 7f4322789000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4106293004' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfe0a368-759e-4b6e-b492-b83b86bd0883"}]: dispatch 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4106293004' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfe0a368-759e-4b6e-b492-b83b86bd0883"}]': finished 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:48:16.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3088758112' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4106293004' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfe0a368-759e-4b6e-b492-b83b86bd0883"}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4106293004' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfe0a368-759e-4b6e-b492-b83b86bd0883"}]': finished 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:48:16.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:16 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3088758112' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:48:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/4106293004' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfe0a368-759e-4b6e-b492-b83b86bd0883"}]: dispatch 2024-09-13T21:48:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/4106293004' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfe0a368-759e-4b6e-b492-b83b86bd0883"}]': finished 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2113751980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:48:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:16 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3088758112' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:48:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:17 smithi114 ceph-mon[23730]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-13T21:48:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T21:48:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:17 smithi114 ceph-mon[23730]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:17 smithi114 ceph-mon[28528]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-13T21:48:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T21:48:17.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:17 smithi114 ceph-mon[28528]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:17 smithi121 ceph-mon[24681]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-13T21:48:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T21:48:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:17 smithi121 ceph-mon[24681]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:19.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:19 smithi114 ceph-mon[23730]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:19.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:19 smithi114 ceph-mon[28528]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:19.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:19 smithi121 ceph-mon[24681]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:21.701 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[23730]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:21.702 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:48:21.702 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:21.702 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[23730]: Deploying daemon osd.0 on smithi114 2024-09-13T21:48:21.702 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[28528]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:21.702 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:48:21.702 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:21.702 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:21 smithi114 ceph-mon[28528]: Deploying daemon osd.0 on smithi114 2024-09-13T21:48:21.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:21 smithi121 ceph-mon[24681]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:21.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:48:21.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:21.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:21 smithi121 ceph-mon[24681]: Deploying daemon osd.0 on smithi114 2024-09-13T21:48:23.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:23 smithi114 ceph-mon[23730]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:23.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:23 smithi114 ceph-mon[28528]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:23.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:23 smithi121 ceph-mon[24681]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:25.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:25 smithi121 ceph-mon[24681]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:26.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:25 smithi114 ceph-mon[23730]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:26.068 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:25 smithi114 ceph-mon[28528]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:27.399 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:48:27] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:28.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:28.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:28.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:28.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:28.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:28.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:28.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:28.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:28.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:28.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:28.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:30 smithi114 ceph-mon[23730]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:30.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:30 smithi114 ceph-mon[28528]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:30.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:30 smithi121 ceph-mon[24681]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:30.373 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 0 on host 'smithi114' 2024-09-13T21:48:30.959 DEBUG:teuthology.orchestra.run.smithi114:osd.0> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T21:48:30.962 INFO:tasks.cephadm:Deploying osd.1 on smithi114 with /dev/vg_nvme/lv_3... 2024-09-13T21:48:30.962 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T21:48:31.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:48:31.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:48:31.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:31.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:31.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:31.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:31.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:31.293 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:48:31.298 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:48:31.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:31.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:31.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:31.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:31.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:31.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:32.048 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:48:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: from='osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:32.326 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[31821]: 2024-09-13T21:48:32.049+0000 7f4a58247700 -1 osd.0 0 waiting for initial osdmap 2024-09-13T21:48:32.326 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[31821]: 2024-09-13T21:48:32.052+0000 7f4a513db700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: from='osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:32.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:32 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:32.352 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:32.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T21:48:32.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T21:48:32.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:32.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: from='osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:32.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:32.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:32.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:32 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:33.191 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[23730]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:48:33.191 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[23730]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T21:48:33.191 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:33.191 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:33.192 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[28528]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:48:33.192 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[28528]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T21:48:33.192 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:33.192 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:33 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:33 smithi121 ceph-mon[24681]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:48:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:33 smithi121 ceph-mon[24681]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T21:48:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:33 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:33 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:33.413 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:48:33.432 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_3 2024-09-13T21:48:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:34 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:48:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:34 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:48:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:34 smithi121 ceph-mon[24681]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:34 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:34 smithi121 ceph-mon[24681]: osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804] boot 2024-09-13T21:48:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:34 smithi121 ceph-mon[24681]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T21:48:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:34 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[28528]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[28528]: osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804] boot 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[28528]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:34.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:48:34.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:48:34.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[23730]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:48:34.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:34.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[23730]: osd.0 [v2:172.21.15.114:6802/965608804,v1:172.21.15.114:6803/965608804] boot 2024-09-13T21:48:34.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[23730]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T21:48:34.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:48:35.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:35 smithi114 ceph-mon[23730]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T21:48:35.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:35 smithi114 ceph-mon[28528]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T21:48:35.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:35 smithi121 ceph-mon[24681]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T21:48:36.309 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[23730]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:36.309 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[23730]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:36.309 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:36.309 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:36.309 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:36.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[28528]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:36.310 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[28528]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:36.310 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:36.310 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:36.310 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:36 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:36 smithi121 ceph-mon[24681]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:36 smithi121 ceph-mon[24681]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:36 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:36 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:36 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:38.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[23730]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:38.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2924871637' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]: dispatch 2024-09-13T21:48:38.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]: dispatch 2024-09-13T21:48:38.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]': finished 2024-09-13T21:48:38.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[23730]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T21:48:38.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:38.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[28528]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:38.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2924871637' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]: dispatch 2024-09-13T21:48:38.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]: dispatch 2024-09-13T21:48:38.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]': finished 2024-09-13T21:48:38.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[28528]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T21:48:38.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:38 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:38.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:38 smithi121 ceph-mon[24681]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:38.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:38 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2924871637' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]: dispatch 2024-09-13T21:48:38.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:38 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]: dispatch 2024-09-13T21:48:38.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:38 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b13c8473-564d-4977-9d35-5e7eccb2320f"}]': finished 2024-09-13T21:48:38.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:38 smithi121 ceph-mon[24681]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T21:48:38.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:38 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:39.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:39 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1021132438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:48:39.247 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:39 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1021132438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:48:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:39 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1021132438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:48:39.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:48:39] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:40.511 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:40.511 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi114 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi114 2024-09-13T21:48:40.512 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.513 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:40.513 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:40.513 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:40.513 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi114 2024-09-13T21:48:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:40.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:40.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:41.614 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:41 smithi114 ceph-mon[23730]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:41.614 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:41 smithi114 ceph-mon[28528]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:41.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:41 smithi121 ceph-mon[24681]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:43.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:43 smithi114 ceph-mon[23730]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:43.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:43 smithi114 ceph-mon[28528]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:43 smithi121 ceph-mon[24681]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:44.640 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:44 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:48:44.640 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:44 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:44.640 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:44 smithi114 ceph-mon[23730]: Deploying daemon osd.1 on smithi114 2024-09-13T21:48:44.640 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:48:44.641 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:44 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:44.641 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:44 smithi114 ceph-mon[28528]: Deploying daemon osd.1 on smithi114 2024-09-13T21:48:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:44 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:48:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:44 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:44 smithi121 ceph-mon[24681]: Deploying daemon osd.1 on smithi114 2024-09-13T21:48:45.654 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:45 smithi114 ceph-mon[23730]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:45.655 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:45 smithi114 ceph-mon[28528]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:45 smithi121 ceph-mon[24681]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:47.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:47 smithi114 ceph-mon[23730]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:47.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:47 smithi114 ceph-mon[28528]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:47.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:47 smithi121 ceph-mon[24681]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[23730]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[28528]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:50 smithi121 ceph-mon[24681]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:50.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:50.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:51.478 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:51 smithi114 ceph-mon[23730]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:51.478 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:51 smithi114 ceph-mon[28528]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:51.729 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:51 smithi121 ceph-mon[24681]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:52.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:48:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:48:52.536 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 1 on host 'smithi114' 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:53.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.030 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.049 DEBUG:teuthology.orchestra.run.smithi114:osd.1> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T21:48:53.052 INFO:tasks.cephadm:Deploying osd.2 on smithi114 with /dev/vg_nvme/lv_2... 2024-09-13T21:48:53.052 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T21:48:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:48:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:48:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[23730]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[23730]: Adjusting osd_memory_target on smithi114 to 7520M 2024-09-13T21:48:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[23730]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T21:48:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[23730]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T21:48:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[23730]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:54.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[28528]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:54.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[28528]: Adjusting osd_memory_target on smithi114 to 7520M 2024-09-13T21:48:54.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[28528]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T21:48:54.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[28528]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T21:48:54.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:54.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-mon[28528]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:54.076 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1[34730]: 2024-09-13T21:48:53.839+0000 7f078c431700 -1 osd.1 0 waiting for initial osdmap 2024-09-13T21:48:54.076 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:48:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1[34730]: 2024-09-13T21:48:53.842+0000 7f0784dc4700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:48:54.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:53 smithi121 ceph-mon[24681]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:54.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:53 smithi121 ceph-mon[24681]: Adjusting osd_memory_target on smithi114 to 7520M 2024-09-13T21:48:54.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:53 smithi121 ceph-mon[24681]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T21:48:54.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:53 smithi121 ceph-mon[24681]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T21:48:54.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:53 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:54.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:53 smithi121 ceph-mon[24681]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:48:55.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:54 smithi121 ceph-mon[24681]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:48:55.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:54 smithi121 ceph-mon[24681]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T21:48:55.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:54 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:55.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:54 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[23730]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[23730]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[28528]: from='osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[28528]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:55.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:54 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:55.627 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:48:55.648 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_2 2024-09-13T21:48:56.100 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:48:56.100 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:48:56.101 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[28528]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:56.101 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[28528]: osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801] boot 2024-09-13T21:48:56.101 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[28528]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T21:48:56.101 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:56.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:48:56.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:48:56.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[23730]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:56.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[23730]: osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801] boot 2024-09-13T21:48:56.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[23730]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T21:48:56.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:55 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:48:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:55 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:48:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:55 smithi121 ceph-mon[24681]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:48:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:55 smithi121 ceph-mon[24681]: osd.1 [v2:172.21.15.114:6810/673244801,v1:172.21.15.114:6811/673244801] boot 2024-09-13T21:48:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:55 smithi121 ceph-mon[24681]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T21:48:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:48:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:56 smithi121 ceph-mon[24681]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T21:48:57.148 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:56 smithi114 ceph-mon[23730]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T21:48:57.148 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:56 smithi114 ceph-mon[28528]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T21:48:57.985 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[28528]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:48:57.985 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[28528]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:57.985 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:57.985 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:57.985 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:57.986 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[23730]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:48:57.986 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[23730]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:57.986 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:57.986 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:57.986 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:48:57 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:57 smithi121 ceph-mon[24681]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:48:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:57 smithi121 ceph-mon[24681]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:57 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:48:58.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:57 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:48:58.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:48:57 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:48:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:48:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi114 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2425673599' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73a72ace-7f75-43c3-aab0-7ca73811182d"}]: dispatch 2024-09-13T21:49:00.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2425673599' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73a72ace-7f75-43c3-aab0-7ca73811182d"}]': finished 2024-09-13T21:49:00.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T21:49:00.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:00.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:00 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1800663258' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi114 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2425673599' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73a72ace-7f75-43c3-aab0-7ca73811182d"}]: dispatch 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2425673599' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73a72ace-7f75-43c3-aab0-7ca73811182d"}]': finished 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1800663258' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:49:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:00.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi114 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2425673599' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73a72ace-7f75-43c3-aab0-7ca73811182d"}]: dispatch 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2425673599' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73a72ace-7f75-43c3-aab0-7ca73811182d"}]': finished 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:00.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:00 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1800663258' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:01 smithi121 ceph-mon[24681]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:49:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[28528]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:49:01.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[23730]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:01.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:01.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:02.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:02 smithi121 ceph-mon[24681]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:02.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:02 smithi121 ceph-mon[24681]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:49:02.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:02 smithi121 ceph-mon[24681]: Cluster is now healthy 2024-09-13T21:49:02.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:02 smithi114 ceph-mon[23730]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:02 smithi114 ceph-mon[23730]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:49:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:02 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T21:49:02.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:02 smithi114 ceph-mon[28528]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:02.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:02 smithi114 ceph-mon[28528]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:49:02.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:02 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T21:49:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:04 smithi121 ceph-mon[24681]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:04.674 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:04 smithi114 ceph-mon[23730]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:04.674 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:04 smithi114 ceph-mon[28528]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:06.618 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:06 smithi114 ceph-mon[23730]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:06.618 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:49:06.618 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:06.618 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:06 smithi114 ceph-mon[28528]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:06.618 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:49:06.618 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:06 smithi121 ceph-mon[24681]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:49:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:07.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:07 smithi121 ceph-mon[24681]: Deploying daemon osd.2 on smithi114 2024-09-13T21:49:07.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:07 smithi114 ceph-mon[23730]: Deploying daemon osd.2 on smithi114 2024-09-13T21:49:07.801 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:07 smithi114 ceph-mon[28528]: Deploying daemon osd.2 on smithi114 2024-09-13T21:49:08.366 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:08 smithi114 ceph-mon[23730]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:08.366 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:08 smithi114 ceph-mon[28528]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:08.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:08 smithi121 ceph-mon[24681]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:10.510 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:10 smithi114 ceph-mon[28528]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:10.511 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:10 smithi114 ceph-mon[23730]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:10.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:10 smithi121 ceph-mon[24681]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:12.201 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:49:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:12.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:12 smithi114 ceph-mon[23730]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:12.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:12.512 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:12.513 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:12 smithi114 ceph-mon[28528]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:12.513 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:12.513 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:12.809 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:12 smithi121 ceph-mon[24681]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:12.809 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:12.810 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:13.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[23730]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[28528]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:13.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:13 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:13 smithi121 ceph-mon[24681]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:13 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:13 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:13 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:13 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:13 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:14.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:49:14.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:49:14.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:49:14.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:49:14.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:49:14.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:49:15.136 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 2 on host 'smithi114' 2024-09-13T21:49:15.699 DEBUG:teuthology.orchestra.run.smithi114:osd.2> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T21:49:15.702 INFO:tasks.cephadm:Deploying osd.3 on smithi114 with /dev/vg_nvme/lv_1... 2024-09-13T21:49:15.702 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: Adjusting osd_memory_target on smithi114 to 5013M 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[23730]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: Adjusting osd_memory_target on smithi114 to 5013M 2024-09-13T21:49:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:49:16.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:16 smithi114 ceph-mon[28528]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:49:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: Adjusting osd_memory_target on smithi114 to 5013M 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:49:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:16 smithi121 ceph-mon[24681]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:49:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[23730]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[23730]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T21:49:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[23730]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T21:49:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[23730]: from='osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[23730]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[28528]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[28528]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T21:49:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[28528]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T21:49:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[28528]: from='osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-mon[28528]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:17.576 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2[37723]: 2024-09-13T21:49:17.467+0000 7f117898e700 -1 osd.2 0 waiting for initial osdmap 2024-09-13T21:49:17.576 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2[37723]: 2024-09-13T21:49:17.475+0000 7f1171321700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:49:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:17 smithi121 ceph-mon[24681]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:17 smithi121 ceph-mon[24681]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T21:49:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:17 smithi121 ceph-mon[24681]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T21:49:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:17 smithi121 ceph-mon[24681]: from='osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:17.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:17 smithi121 ceph-mon[24681]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:18.278 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:49:18.300 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi114:vg_nvme/lv_1 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[28528]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[28528]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[23730]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[23730]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T21:49:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:18 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:18 smithi121 ceph-mon[24681]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:49:18.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:18 smithi121 ceph-mon[24681]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T21:49:18.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:18 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:18 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:18.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:18 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:19.683 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:49:19.683 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:49:19.683 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[28528]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:19.683 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[28528]: osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857] boot 2024-09-13T21:49:19.683 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[28528]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T21:49:19.683 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:19.683 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T21:49:19.684 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:49:19.684 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:49:19.684 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[23730]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:19.684 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[23730]: osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857] boot 2024-09-13T21:49:19.684 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[23730]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T21:49:19.684 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:19.684 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:19 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T21:49:19.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:19 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:49:19.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:19 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:49:19.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:19 smithi121 ceph-mon[24681]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:49:19.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:19 smithi121 ceph-mon[24681]: osd.2 [v2:172.21.15.114:6818/4280964857,v1:172.21.15.114:6819/4280964857] boot 2024-09-13T21:49:19.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:19 smithi121 ceph-mon[24681]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T21:49:19.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:19 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:49:19.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:19 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T21:49:20.788 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:49:20] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[23730]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[23730]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[28528]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T21:49:20.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T21:49:20.790 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[28528]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:20.790 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:49:20.790 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:49:20.790 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:20.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T21:49:20.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:20 smithi121 ceph-mon[24681]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T21:49:20.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T21:49:20.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:20 smithi121 ceph-mon[24681]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi114:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:20.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:49:20.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:49:20.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:21.819 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:49:21 smithi114 sudo[38973]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:49:21.819 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:49:21 smithi114 sudo[38973]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:21.819 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:49:21 smithi114 sudo[38973]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi114 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:21.820 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi114 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:21.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:21 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:21.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T21:49:21.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi114 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:21.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:21 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:49:21 smithi114 sudo[39002]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:49:22.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:49:21 smithi114 sudo[39002]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:22.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 21:49:21 smithi114 sudo[39002]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:22.528 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39122]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:49:22.529 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39122]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:22.529 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39122]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:22.529 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:49:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:22.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:49:22.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3463220088' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf107b20-067f-407b-bb0e-0e09e4e03029"}]: dispatch 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3463220088' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf107b20-067f-407b-bb0e-0e09e4e03029"}]': finished 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T21:49:22.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3463220088' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf107b20-067f-407b-bb0e-0e09e4e03029"}]: dispatch 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3463220088' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf107b20-067f-407b-bb0e-0e09e4e03029"}]': finished 2024-09-13T21:49:22.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T21:49:22.532 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:22.780 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39131]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:49:22.781 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39131]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:22.781 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39131]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:22.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3463220088' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf107b20-067f-407b-bb0e-0e09e4e03029"}]: dispatch 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3463220088' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf107b20-067f-407b-bb0e-0e09e4e03029"}]': finished 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T21:49:22.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:23.043 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39163]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:49:23.043 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39163]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:23.043 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:22 smithi114 sudo[39163]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:23.294 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39174]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:49:23.294 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39174]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:23.294 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39174]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:23.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39229]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:49:23.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39229]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:23.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39229]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:23.805 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39250]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:49:23.806 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39250]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:23.806 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39250]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:23.806 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:23 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2098212888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:23.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:23 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2098212888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:24.069 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39439]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:49:24.069 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39439]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:24.069 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:23 smithi114 sudo[39439]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:24.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 sudo[39630]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:49:24.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 sudo[39630]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:24.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 sudo[39630]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:24.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2098212888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:24.562 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 sudo[27166]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:49:24.563 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 sudo[27166]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:24.563 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 sudo[27166]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:24.563 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 sudo[27169]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:49:24.563 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 sudo[27169]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:49:24.563 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 sudo[27169]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T21:49:24.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[28528]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:49:24.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:24 smithi114 ceph-mon[23730]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:24.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:24.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: Cluster is now healthy 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:49:24.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:49:24.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:49:24.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:24 smithi121 ceph-mon[24681]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:25.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:25 smithi114 ceph-mon[23730]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:25.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:25 smithi114 ceph-mon[23730]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:25.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:25 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:25.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:25 smithi114 ceph-mon[28528]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:25.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:25 smithi114 ceph-mon[28528]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:25.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:25 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:25.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:25 smithi121 ceph-mon[24681]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:49:25.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:25 smithi121 ceph-mon[24681]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:49:25.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:25 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:26.720 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:26 smithi114 ceph-mon[23730]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:26.720 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:26 smithi114 ceph-mon[28528]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:26.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:26 smithi121 ceph-mon[24681]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:29 smithi114 ceph-mon[23730]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:29.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:29 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:49:29.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:29 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:29 smithi114 ceph-mon[28528]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:29 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:49:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:29 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:29 smithi121 ceph-mon[24681]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:29 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:49:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:29 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:30 smithi121 ceph-mon[24681]: Deploying daemon osd.3 on smithi114 2024-09-13T21:49:30.628 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:30 smithi114 ceph-mon[23730]: Deploying daemon osd.3 on smithi114 2024-09-13T21:49:30.629 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:30 smithi114 ceph-mon[28528]: Deploying daemon osd.3 on smithi114 2024-09-13T21:49:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:31 smithi121 ceph-mon[24681]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:31.647 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:31 smithi114 ceph-mon[23730]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:31.647 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:31 smithi114 ceph-mon[28528]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:32.261 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:49:32] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:33.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:33 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:33.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:33 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:33.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:33 smithi121 ceph-mon[24681]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:33.220 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:33 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:33.220 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:33 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:33.220 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:33 smithi114 ceph-mon[23730]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:33.221 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:33 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:33.221 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:33 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:33.221 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:33 smithi114 ceph-mon[28528]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:35 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:35 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:35 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:35 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:35 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:35 smithi121 ceph-mon[24681]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[23730]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:35.501 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:35.502 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:35.502 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:35.502 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:35.502 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:35 smithi114 ceph-mon[28528]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:36.721 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:36 smithi114 ceph-mon[23730]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:36.722 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:36 smithi114 ceph-mon[28528]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:36.751 INFO:teuthology.orchestra.run.smithi114.stdout:Created osd(s) 3 on host 'smithi114' 2024-09-13T21:49:36.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:36 smithi121 ceph-mon[24681]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:37.348 DEBUG:teuthology.orchestra.run.smithi114:osd.3> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T21:49:37.350 INFO:tasks.cephadm:Deploying osd.4 on smithi121 with /dev/vg_nvme/lv_4... 2024-09-13T21:49:37.350 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T21:49:37.988 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: Adjusting osd_memory_target on smithi114 to 3760M 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:37.989 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:37 smithi121 ceph-mon[24681]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:49:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: Adjusting osd_memory_target on smithi114 to 3760M 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[23730]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: Adjusting osd_memory_target on smithi114 to 3760M 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:37 smithi114 ceph-mon[28528]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:49:38.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:38 smithi121 ceph-mon[24681]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T21:49:38.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:38 smithi121 ceph-mon[24681]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T21:49:38.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:38 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:38.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:38 smithi121 ceph-mon[24681]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:38.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:38 smithi121 ceph-mon[24681]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[23730]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T21:49:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[23730]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[23730]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[23730]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[28528]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[28528]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[28528]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T21:49:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-mon[28528]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:49:39.076 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T21:49:38.791+0000 7fbc85cd7700 -1 osd.3 0 waiting for initial osdmap 2024-09-13T21:49:39.076 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 21:49:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T21:49:38.795+0000 7fbc82672700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:49:39.885 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:49:39.903 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_4 2024-09-13T21:49:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[23730]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:49:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[23730]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T21:49:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[28528]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:49:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[28528]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T21:49:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:39 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:39 smithi121 ceph-mon[24681]: from='osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]': finished 2024-09-13T21:49:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:39 smithi121 ceph-mon[24681]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T21:49:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:39 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:39 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:41.040 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[23730]: osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472] boot 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[23730]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[23730]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[28528]: osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472] boot 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[28528]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T21:49:41.041 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:41.042 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:40 smithi114 ceph-mon[28528]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:41.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:40 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:49:41.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:40 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:49:41.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:40 smithi121 ceph-mon[24681]: osd.3 [v2:172.21.15.114:6826/575349472,v1:172.21.15.114:6827/575349472] boot 2024-09-13T21:49:41.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:40 smithi121 ceph-mon[24681]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T21:49:41.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:41.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:40 smithi121 ceph-mon[24681]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:41.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:49:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[23730]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[28528]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:41.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:41.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:49:41.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:49:41.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:41 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:42.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:41 smithi121 ceph-mon[24681]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T21:49:42.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:41 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:42.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:41 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:42.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:41 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:49:42.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:41 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:49:42.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:41 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:43.050 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:42 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi114 2024-09-13T21:49:43.050 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:42 smithi121 ceph-mon[24681]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:43.050 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:42 smithi121 ceph-mon[24681]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:42 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi114 2024-09-13T21:49:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:42 smithi114 ceph-mon[23730]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:42 smithi114 ceph-mon[23730]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:43.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:42 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi114 2024-09-13T21:49:43.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:42 smithi114 ceph-mon[28528]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:43.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:42 smithi114 ceph-mon[28528]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:44.245 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:43 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/287596014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]: dispatch 2024-09-13T21:49:44.245 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:43 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]: dispatch 2024-09-13T21:49:44.245 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:43 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]': finished 2024-09-13T21:49:44.245 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:43 smithi121 ceph-mon[24681]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T21:49:44.245 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:43 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:49:44.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/287596014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]: dispatch 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]: dispatch 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]': finished 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[23730]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/287596014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]: dispatch 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]: dispatch 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2921ec7-d207-4918-9f9e-5c81f71d12ff"}]': finished 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[28528]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T21:49:44.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:43 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:49:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:44 smithi121 ceph-mon[24681]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:44 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/3977617073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:45.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:44 smithi114 ceph-mon[23730]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:45.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:44 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/3977617073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:45.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:44 smithi114 ceph-mon[28528]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:45.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:44 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/3977617073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:49:46.625 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:46 smithi114 ceph-mon[23730]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:46.625 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:46 smithi114 ceph-mon[28528]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:46.717 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:46 smithi121 ceph-mon[24681]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:49.608 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:49 smithi121 ceph-mon[24681]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:49.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:49 smithi114 ceph-mon[23730]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:49.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:49 smithi114 ceph-mon[28528]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:50.528 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:49:50.528 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:50.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:49:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:50.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:49:50.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:51.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:51 smithi121 ceph-mon[24681]: Deploying daemon osd.4 on smithi121 2024-09-13T21:49:51.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:51 smithi121 ceph-mon[24681]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:51.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:51 smithi114 ceph-mon[23730]: Deploying daemon osd.4 on smithi121 2024-09-13T21:49:51.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:51 smithi114 ceph-mon[23730]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:51.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:51 smithi114 ceph-mon[28528]: Deploying daemon osd.4 on smithi121 2024-09-13T21:49:51.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:51 smithi114 ceph-mon[28528]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:52.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:49:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:53 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:53.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:53 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:53.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:53 smithi121 ceph-mon[24681]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:53.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:53 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:53.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:53 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:53.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:53 smithi114 ceph-mon[23730]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:53 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:53 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:53 smithi114 ceph-mon[28528]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:54.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:54 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:49:54] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:49:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:55 smithi121 ceph-mon[24681]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[23730]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[28528]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:55.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:55.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:55.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:55.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:56.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[28528]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[28528]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[28528]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[28528]: Reconfiguring osd.4 (unknown last config time)... 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[28528]: Reconfiguring daemon osd.4 on smithi121 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[28528]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[23730]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[23730]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[23730]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[23730]: Reconfiguring osd.4 (unknown last config time)... 2024-09-13T21:49:56.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[23730]: Reconfiguring daemon osd.4 on smithi121 2024-09-13T21:49:56.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[23730]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T21:49:56.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:56 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:56.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:56 smithi121 ceph-mon[24681]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:56 smithi121 ceph-mon[24681]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:56 smithi121 ceph-mon[24681]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:56.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:56 smithi121 ceph-mon[24681]: Reconfiguring osd.4 (unknown last config time)... 2024-09-13T21:49:56.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:56 smithi121 ceph-mon[24681]: Reconfiguring daemon osd.4 on smithi121 2024-09-13T21:49:56.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:56 smithi121 ceph-mon[24681]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T21:49:56.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:56 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:58.477 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:58 smithi121 ceph-mon[24681]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:58.477 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:58.477 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:58.477 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:58.478 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:58.478 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:58.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[23730]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[28528]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:58.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:49:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:59 smithi121 ceph-mon[24681]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:59 smithi121 ceph-mon[24681]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T21:49:59.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:59 smithi121 ceph-mon[24681]: Cluster is now healthy 2024-09-13T21:49:59.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:59 smithi121 ceph-mon[24681]: from='osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:49:59.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:49:59 smithi121 ceph-mon[24681]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:49:59.736 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[23730]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:59.736 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[23730]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[23730]: from='osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[23730]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[28528]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[28528]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[28528]: from='osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:49:59.737 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-mon[28528]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:50:00.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:49:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:49:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: from='osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:50:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[23730]: overall HEALTH_OK 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: from='osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:50:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:00 smithi114 ceph-mon[28528]: overall HEALTH_OK 2024-09-13T21:50:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T21:50:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T21:50:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: from='osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:50:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:00 smithi121 ceph-mon[24681]: overall HEALTH_OK 2024-09-13T21:50:01.448 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 4 on host 'smithi121' 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:02.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.120 DEBUG:teuthology.orchestra.run.smithi121:osd.4> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T21:50:02.123 INFO:tasks.cephadm:Deploying osd.5 on smithi121 with /dev/vg_nvme/lv_3... 2024-09-13T21:50:02.123 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T21:50:02.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:02.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:50:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:50:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: Adjusting osd_memory_target on smithi121 to 16063M 2024-09-13T21:50:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788] boot 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:02 smithi121 ceph-mon[24681]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: Adjusting osd_memory_target on smithi121 to 16063M 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:50:03.122 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788] boot 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[23730]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: Adjusting osd_memory_target on smithi121 to 16063M 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: osd.4 [v2:172.21.15.121:6800/3259223788,v1:172.21.15.121:6801/3259223788] boot 2024-09-13T21:50:03.123 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T21:50:03.124 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:03.124 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:02 smithi114 ceph-mon[28528]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T21:50:04.730 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:50:04.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_3 2024-09-13T21:50:05.034 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:04 smithi121 ceph-mon[24681]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:05.034 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:04 smithi121 ceph-mon[24681]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T21:50:05.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:04 smithi114 ceph-mon[23730]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:05.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:04 smithi114 ceph-mon[23730]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T21:50:05.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:04 smithi114 ceph-mon[28528]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:05.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:04 smithi114 ceph-mon[28528]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T21:50:05.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:05 smithi121 ceph-mon[24681]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T21:50:06.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:05 smithi114 ceph-mon[23730]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T21:50:06.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:05 smithi114 ceph-mon[28528]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T21:50:07.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:06 smithi121 ceph-mon[24681]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:50:07.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:06 smithi121 ceph-mon[24681]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T21:50:07.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:07.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:07.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:07.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[23730]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[23730]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[28528]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[28528]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:07.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:08.167 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:07 smithi121 ceph-mon[24681]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:08.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:07 smithi114 ceph-mon[23730]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:08.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:07 smithi114 ceph-mon[28528]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:09.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[23730]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/3827364904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]: dispatch 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]: dispatch 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]': finished 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[23730]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[28528]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/3827364904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]: dispatch 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]: dispatch 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]': finished 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[28528]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T21:50:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:08 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:09.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:08 smithi121 ceph-mon[24681]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-13T21:50:09.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:08 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/3827364904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]: dispatch 2024-09-13T21:50:09.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:08 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]: dispatch 2024-09-13T21:50:09.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:08 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d136ab9a-1360-4cf1-a932-b57c1e59436a"}]': finished 2024-09-13T21:50:09.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:08 smithi121 ceph-mon[24681]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T21:50:09.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:08 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:10.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:09 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/2512180461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:50:10.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:09 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/2512180461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:50:10.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:09 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/2512180461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:50:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:10 smithi114 ceph-mon[23730]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T21:50:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:10 smithi114 ceph-mon[28528]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T21:50:11.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:10 smithi121 ceph-mon[24681]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T21:50:13.093 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:12 smithi121 ceph-mon[24681]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T21:50:13.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:12 smithi114 ceph-mon[23730]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T21:50:13.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:12 smithi114 ceph-mon[28528]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T21:50:15.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:50:14] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:14 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:15.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:50:15.170 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:50:15.171 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.171 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:15.171 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:15.171 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:15.171 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:15.171 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:14 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:16.223 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi121 2024-09-13T21:50:16.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:50:16.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:16.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: Deploying daemon osd.5 on smithi121 2024-09-13T21:50:16.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:15 smithi121 ceph-mon[24681]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T21:50:16.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:50:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:50:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi121 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: Deploying daemon osd.5 on smithi121 2024-09-13T21:50:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[23730]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi121 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: Deploying daemon osd.5 on smithi121 2024-09-13T21:50:16.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:15 smithi114 ceph-mon[28528]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T21:50:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[23730]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[28528]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:17.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:17 smithi121 ceph-mon[24681]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:17.291 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:17.291 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:17.291 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:17.291 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:17.291 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:18.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:18 smithi114 ceph-mon[23730]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T21:50:18.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:18 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T21:50:18.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:18 smithi114 ceph-mon[28528]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T21:50:18.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:18 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T21:50:18.329 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:18 smithi121 ceph-mon[24681]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T21:50:18.329 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:18 smithi121 ceph-mon[24681]: Cluster is now healthy 2024-09-13T21:50:19.309 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:19 smithi121 ceph-mon[24681]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:19 smithi114 ceph-mon[23730]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:19 smithi114 ceph-mon[28528]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:20.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:50:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:50:19] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:21.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:21.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:21.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:20 smithi121 ceph-mon[24681]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:20 smithi114 ceph-mon[23730]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:20 smithi114 ceph-mon[28528]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:22 smithi121 ceph-mon[24681]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:22 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[23730]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[28528]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:23.239 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:23.240 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:23.240 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:22 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:25.272 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:25 smithi121 ceph-mon[24681]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:25.306 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:25 smithi114 ceph-mon[23730]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:25.306 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:25 smithi114 ceph-mon[28528]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:25.597 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 5 on host 'smithi121' 2024-09-13T21:50:26.229 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:26.229 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.230 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:26 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.290 DEBUG:teuthology.orchestra.run.smithi121:osd.5> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T21:50:26.293 INFO:tasks.cephadm:Deploying osd.6 on smithi121 with /dev/vg_nvme/lv_2... 2024-09-13T21:50:26.293 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T21:50:26.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:26.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:26.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:26.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:26.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:26.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:26.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:26.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:26.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:26 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-mon[24681]: Adjusting osd_memory_target on smithi121 to 8031M 2024-09-13T21:50:27.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-mon[24681]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:27.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-mon[24681]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T21:50:27.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-mon[24681]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T21:50:27.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:27.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-mon[24681]: from='osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:27.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-mon[24681]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:27.372 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T21:50:27.078+0000 7fc8afb8a700 -1 osd.5 0 waiting for initial osdmap 2024-09-13T21:50:27.372 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 21:50:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T21:50:27.085+0000 7fc8a9d20700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:50:27.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[28528]: Adjusting osd_memory_target on smithi121 to 8031M 2024-09-13T21:50:27.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[28528]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:27.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[28528]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T21:50:27.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[28528]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[28528]: from='osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[28528]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[23730]: Adjusting osd_memory_target on smithi121 to 8031M 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[23730]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[23730]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[23730]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[23730]: from='osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:27.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:27 smithi114 ceph-mon[23730]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:28.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:28 smithi121 ceph-mon[24681]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:28.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:28 smithi121 ceph-mon[24681]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T21:50:28.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:28 smithi121 ceph-mon[24681]: osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015] boot 2024-09-13T21:50:28.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:28 smithi121 ceph-mon[24681]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T21:50:28.350 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[23730]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:28.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[23730]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T21:50:28.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[23730]: osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015] boot 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[23730]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[28528]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[28528]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[28528]: osd.5 [v2:172.21.15.121:6808/2151936015,v1:172.21.15.121:6809/2151936015] boot 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[28528]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T21:50:28.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:28.869 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:50:28.888 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_2 2024-09-13T21:50:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:29 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:50:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:29 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:50:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:29 smithi121 ceph-mon[24681]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:29 smithi121 ceph-mon[24681]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T21:50:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:50:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:50:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[23730]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[23730]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T21:50:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:50:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:50:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[28528]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:50:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:29 smithi114 ceph-mon[28528]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T21:50:31.211 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:31 smithi121 ceph-mon[24681]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:31.211 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:31 smithi121 ceph-mon[24681]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T21:50:31.211 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:31 smithi121 ceph-mon[24681]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:50:31.211 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:31 smithi121 ceph-mon[24681]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T21:50:31.211 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:31.211 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:31.212 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:31 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:31.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[23730]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:31.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[23730]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T21:50:31.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[23730]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[23730]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[28528]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[28528]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[28528]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[28528]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:31 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:32.330 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:32 smithi121 ceph-mon[24681]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:32.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:32 smithi121 ceph-mon[24681]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T21:50:32.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:32 smithi114 ceph-mon[23730]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:32.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:32 smithi114 ceph-mon[23730]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T21:50:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:32 smithi114 ceph-mon[28528]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:32 smithi114 ceph-mon[28528]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T21:50:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:33 smithi121 ceph-mon[24681]: pgmap v139: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:33.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:33 smithi114 ceph-mon[23730]: pgmap v139: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:33.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:33 smithi114 ceph-mon[28528]: pgmap v139: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:34.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:34 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/3024098134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]: dispatch 2024-09-13T21:50:34.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:34 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]: dispatch 2024-09-13T21:50:34.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:34 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]': finished 2024-09-13T21:50:34.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:34 smithi121 ceph-mon[24681]: osdmap e43: 7 total, 6 up, 7 in 2024-09-13T21:50:34.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:34 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:34.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:34 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/4270485308' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:50:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/3024098134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]: dispatch 2024-09-13T21:50:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]: dispatch 2024-09-13T21:50:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]': finished 2024-09-13T21:50:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[23730]: osdmap e43: 7 total, 6 up, 7 in 2024-09-13T21:50:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:34.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/4270485308' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:50:34.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/3024098134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]: dispatch 2024-09-13T21:50:34.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]: dispatch 2024-09-13T21:50:34.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c"}]': finished 2024-09-13T21:50:34.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[28528]: osdmap e43: 7 total, 6 up, 7 in 2024-09-13T21:50:34.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:34.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:34 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/4270485308' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:50:35.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:35 smithi121 ceph-mon[24681]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:50:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:35 smithi121 ceph-mon[24681]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T21:50:35.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:35 smithi121 ceph-mon[24681]: Cluster is now healthy 2024-09-13T21:50:35.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:50:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:50:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:35.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:35 smithi114 ceph-mon[28528]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:50:35.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:35 smithi114 ceph-mon[28528]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T21:50:35.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:35 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T21:50:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:35 smithi114 ceph-mon[23730]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:50:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:35 smithi114 ceph-mon[23730]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T21:50:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:35 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T21:50:36.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:36 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:36.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:36 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi121 2024-09-13T21:50:36.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:36 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:36.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:36.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:36 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi121 2024-09-13T21:50:36.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:36 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:36 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:36 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi121 2024-09-13T21:50:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:36 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:37.337 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:37 smithi121 ceph-mon[24681]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T21:50:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:37 smithi114 ceph-mon[23730]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T21:50:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:37 smithi114 ceph-mon[28528]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T21:50:39.340 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:39 smithi121 ceph-mon[24681]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T21:50:39.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:39 smithi114 ceph-mon[23730]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T21:50:39.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:39 smithi114 ceph-mon[28528]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T21:50:40.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:50:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:50:39] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:41.242 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:41.242 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:41.242 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:40 smithi121 ceph-mon[24681]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T21:50:41.242 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:50:41.242 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:40 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:41.242 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:40 smithi121 ceph-mon[24681]: Deploying daemon osd.6 on smithi121 2024-09-13T21:50:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[23730]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[23730]: Deploying daemon osd.6 on smithi121 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[28528]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T21:50:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:50:41.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:41.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:40 smithi114 ceph-mon[28528]: Deploying daemon osd.6 on smithi121 2024-09-13T21:50:43.297 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:43 smithi121 ceph-mon[24681]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:50:43.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:43 smithi114 ceph-mon[23730]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:50:43.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:43 smithi114 ceph-mon[28528]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:50:45.195 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:45 smithi121 ceph-mon[24681]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:45.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:45 smithi114 ceph-mon[23730]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:45.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:45 smithi114 ceph-mon[28528]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:46.751 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:46 smithi121 ceph-mon[24681]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:46.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:46 smithi114 ceph-mon[23730]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:46 smithi114 ceph-mon[28528]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:48.077 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:47 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:48.077 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:47 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:48.077 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:47 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:48.077 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:47 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:48.077 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:47 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:48.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:48.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:48.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:47 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:48.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:48 smithi114 ceph-mon[23730]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:48.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:48 smithi114 ceph-mon[28528]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:48 smithi121 ceph-mon[24681]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:49.522 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 6 on host 'smithi121' 2024-09-13T21:50:50.183 DEBUG:teuthology.orchestra.run.smithi121:osd.6> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T21:50:50.186 INFO:tasks.cephadm:Deploying osd.7 on smithi121 with /dev/vg_nvme/lv_1... 2024-09-13T21:50:50.186 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: Adjusting osd_memory_target on smithi121 to 5354M 2024-09-13T21:50:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:50:50.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:50:50.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[23730]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:50.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:50.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: Adjusting osd_memory_target on smithi121 to 5354M 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:50:50.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:50 smithi114 ceph-mon[28528]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:50.830 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.830 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: Adjusting osd_memory_target on smithi121 to 5354M 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:50.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:50:50.832 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:50:50.832 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:50 smithi121 ceph-mon[24681]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:51.726 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:51 smithi121 ceph-mon[24681]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T21:50:51.726 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:51 smithi121 ceph-mon[24681]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T21:50:51.726 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:51 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:51.727 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:51 smithi121 ceph-mon[24681]: from='osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:51.727 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:51 smithi121 ceph-mon[24681]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:51.727 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 21:50:51 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T21:50:51.586+0000 7f9cb09d2700 -1 osd.6 0 waiting for initial osdmap 2024-09-13T21:50:51.727 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 21:50:51 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T21:50:51.592+0000 7f9ca9365700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[23730]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[23730]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[23730]: from='osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[23730]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[28528]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[28528]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[28528]: from='osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:52.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:51 smithi114 ceph-mon[28528]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:50:52.791 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:52 smithi121 ceph-mon[24681]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:52.791 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:52 smithi121 ceph-mon[24681]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T21:50:52.791 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:52.791 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:52 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:52.791 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:52 smithi121 ceph-mon[24681]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:52.796 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-09-13T21:50:52.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_1 2024-09-13T21:50:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[23730]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[23730]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[23730]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[28528]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[28528]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:52 smithi114 ceph-mon[28528]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:50:53.782 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:53 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:50:53.782 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:53 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:50:53.782 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:53 smithi121 ceph-mon[24681]: osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710] boot 2024-09-13T21:50:53.782 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:53 smithi121 ceph-mon[24681]: osdmap e46: 7 total, 7 up, 7 in 2024-09-13T21:50:53.782 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:53 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:53.782 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:53 smithi121 ceph-mon[24681]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T21:50:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[23730]: osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710] boot 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[23730]: osdmap e46: 7 total, 7 up, 7 in 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[23730]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[28528]: osd.6 [v2:172.21.15.121:6816/877204710,v1:172.21.15.121:6817/877204710] boot 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[28528]: osdmap e46: 7 total, 7 up, 7 in 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:50:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:53 smithi114 ceph-mon[28528]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T21:50:54.615 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:54 smithi121 ceph-mon[24681]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:55.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:54 smithi114 ceph-mon[23730]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:54 smithi114 ceph-mon[28528]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:55.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:50:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='client.24305 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi121 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:55.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:55.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:55.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:55.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:55.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:55 smithi121 ceph-mon[24681]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T21:50:56.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='client.24305 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi121 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[28528]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='client.24305 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi121 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:56.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:56.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:55 smithi114 ceph-mon[23730]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T21:50:56.686 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:56 smithi121 ceph-mon[24681]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:50:56.686 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:56 smithi121 ceph-mon[24681]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T21:50:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:56 smithi114 ceph-mon[23730]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:50:57.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:56 smithi114 ceph-mon[23730]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T21:50:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:56 smithi114 ceph-mon[28528]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-09-13T21:50:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:56 smithi114 ceph-mon[28528]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T21:50:57.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:57 smithi121 ceph-mon[24681]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:58.026 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:57 smithi114 ceph-mon[23730]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:58.027 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:57 smithi114 ceph-mon[28528]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:50:58.027 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:50:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:50:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:50:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/3934919428' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]: dispatch 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]: dispatch 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]': finished 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: osdmap e51: 8 total, 7 up, 8 in 2024-09-13T21:50:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/3934919428' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]: dispatch 2024-09-13T21:50:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]: dispatch 2024-09-13T21:50:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]': finished 2024-09-13T21:50:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: osdmap e51: 8 total, 7 up, 8 in 2024-09-13T21:50:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:58 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:50:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/3934919428' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]: dispatch 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]: dispatch 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40188450-fd99-40b6-a68d-19e286af2201"}]': finished 2024-09-13T21:50:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: osdmap e51: 8 total, 7 up, 8 in 2024-09-13T21:50:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:58 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:00.027 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:59 smithi114 ceph-mon[23730]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:51:00.027 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:59 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T21:51:00.027 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:50:59 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/142725166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:51:00.027 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:59 smithi114 ceph-mon[28528]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:51:00.027 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:59 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T21:51:00.028 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:50:59 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/142725166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:51:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:59 smithi121 ceph-mon[24681]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-09-13T21:51:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:59 smithi121 ceph-mon[24681]: Cluster is now healthy 2024-09-13T21:51:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:50:59 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/142725166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:51:00.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:00 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:51:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:51:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:01 smithi114 ceph-mon[23730]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-13T21:51:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:01 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:01 smithi114 ceph-mon[28528]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-13T21:51:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:01 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:01 smithi121 ceph-mon[24681]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-13T21:51:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:01 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:03.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:03 smithi121 ceph-mon[24681]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-13T21:51:03.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:03 smithi114 ceph-mon[28528]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-13T21:51:03.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:03 smithi114 ceph-mon[23730]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-13T21:51:05.249 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:05 smithi114 ceph-mon[23730]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T21:51:05.250 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:05 smithi114 ceph-mon[28528]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T21:51:05.293 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:05 smithi121 ceph-mon[24681]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T21:51:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:06 smithi121 ceph-mon[24681]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T21:51:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:51:06.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:06 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:06.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:06 smithi121 ceph-mon[24681]: Deploying daemon osd.7 on smithi121 2024-09-13T21:51:06.773 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[23730]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T21:51:06.773 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:51:06.773 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:06.773 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[23730]: Deploying daemon osd.7 on smithi121 2024-09-13T21:51:06.773 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[28528]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T21:51:06.773 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:51:06.773 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:06.774 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:06 smithi114 ceph-mon[28528]: Deploying daemon osd.7 on smithi121 2024-09-13T21:51:08.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:08 smithi121 ceph-mon[24681]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:09.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:08 smithi114 ceph-mon[23730]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:08 smithi114 ceph-mon[28528]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:11.113 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:10 smithi121 ceph-mon[24681]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:10 smithi114 ceph-mon[23730]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:10 smithi114 ceph-mon[28528]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:12.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:12 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:12.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:12.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:12 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:13.449 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:13 smithi121 ceph-mon[24681]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:13.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:13 smithi114 ceph-mon[23730]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:13.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:13 smithi114 ceph-mon[28528]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:14.159 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 7 on host 'smithi121' 2024-09-13T21:51:14.780 DEBUG:teuthology.orchestra.run.smithi121:osd.7> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T21:51:14.782 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T21:51:14.782 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T21:51:15.399 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: Adjusting osd_memory_target on smithi121 to 4015M 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.400 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.401 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: Adjusting osd_memory_target on smithi121 to 4015M 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:51:15.402 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:15 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:51:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: Adjusting osd_memory_target on smithi121 to 4015M 2024-09-13T21:51:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:51:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:51:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:51:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:15 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:51:16.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[28528]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T21:51:16.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[28528]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T21:51:16.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:16.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[28528]: from='osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:51:16.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[28528]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:51:16.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[23730]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T21:51:16.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[23730]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T21:51:16.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:16.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[23730]: from='osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:51:16.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:16 smithi114 ceph-mon[23730]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:51:16.562 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:16 smithi121 ceph-mon[24681]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T21:51:16.651 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:16 smithi121 ceph-mon[24681]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T21:51:16.651 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:16 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:16.651 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:16 smithi121 ceph-mon[24681]: from='osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:51:16.652 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:16 smithi121 ceph-mon[24681]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-09-13T21:51:16.652 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 21:51:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T21:51:16.212+0000 7f94ee721700 -1 osd.7 0 waiting for initial osdmap 2024-09-13T21:51:16.652 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 21:51:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T21:51:16.218+0000 7f94e90b8700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:51:16.724 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:51:17.189 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1726264252,"num_in_osds":8,"osd_in_since":1726264258,"num_remapped_pgs":0} 2024-09-13T21:51:17.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:51:17] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3862316818' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:51:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T21:51:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3862316818' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:51:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:17 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:51:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-09-13T21:51:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T21:51:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3862316818' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:51:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:17 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:18.191 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[28528]: purged_snaps scrub starts 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[28528]: purged_snaps scrub ok 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi121 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[28528]: osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357] boot 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[28528]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[23730]: purged_snaps scrub starts 2024-09-13T21:51:18.494 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[23730]: purged_snaps scrub ok 2024-09-13T21:51:18.495 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi121 2024-09-13T21:51:18.495 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[23730]: osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357] boot 2024-09-13T21:51:18.495 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[23730]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T21:51:18.495 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:18 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:18.520 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:18 smithi121 ceph-mon[24681]: purged_snaps scrub starts 2024-09-13T21:51:18.520 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:18 smithi121 ceph-mon[24681]: purged_snaps scrub ok 2024-09-13T21:51:18.520 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:18 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi121 2024-09-13T21:51:18.520 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:18 smithi121 ceph-mon[24681]: osd.7 [v2:172.21.15.121:6824/3388831357,v1:172.21.15.121:6825/3388831357] boot 2024-09-13T21:51:18.521 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:18 smithi121 ceph-mon[24681]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T21:51:18.521 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:18 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:19.466 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:19 smithi121 ceph-mon[24681]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:19.467 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:19 smithi121 ceph-mon[24681]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T21:51:19.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:19 smithi114 ceph-mon[23730]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:19.506 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:19 smithi114 ceph-mon[23730]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T21:51:19.506 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:19 smithi114 ceph-mon[28528]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:19.506 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:19 smithi114 ceph-mon[28528]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T21:51:20.085 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:51:20.558 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:20 smithi121 ceph-mon[24681]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T21:51:20.559 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:20 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/898871773' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:51:20.559 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:20.559 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:20 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:20.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:51:20] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:51:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[28528]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T21:51:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/898871773' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:51:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[23730]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T21:51:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/898871773' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:51:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:20 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:20.712 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1726264277,"num_in_osds":8,"osd_in_since":1726264258,"num_remapped_pgs":1} 2024-09-13T21:51:20.713 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T21:51:20.919 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:51:21.308 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:21 smithi114 ceph-mon[23730]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:21.308 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:21 smithi114 ceph-mon[23730]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T21:51:21.308 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:21 smithi114 ceph-mon[28528]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:21.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:21 smithi114 ceph-mon[28528]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T21:51:21.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:21 smithi121 ceph-mon[24681]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:21.593 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:21 smithi121 ceph-mon[24681]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T21:51:22.446 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:51:22.446 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":58,"fsid":"8ccc9cd8-7219-11ef-bceb-c7b262605968","created":"2024-09-13T21:46:19.540411+0000","modified":"2024-09-13T21:51:21.487556+0000","last_up_change":"2024-09-13T21:51:17.202594+0000","last_in_change":"2024-09-13T21:50:58.692065+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T21:49:18.809652+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"bfe0a368-759e-4b6e-b492-b83b86bd0883","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6803","nonce":965608804}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6805","nonce":965608804}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6809","nonce":965608804}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6807","nonce":965608804}]},"public_addr":"172.21.15.114:6803/965608804","cluster_addr":"172.21.15.114:6805/965608804","heartbeat_back_addr":"172.21.15.114:6809/965608804","heartbeat_front_addr":"172.21.15.114:6807/965608804","state":["exists","up"]},{"osd":1,"uuid":"b13c8473-564d-4977-9d35-5e7eccb2320f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6811","nonce":673244801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6813","nonce":673244801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6817","nonce":673244801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6815","nonce":673244801}]},"public_addr":"172.21.15.114:6811/673244801","cluster_addr":"172.21.15.114:6813/673244801","heartbeat_back_addr":"172.21.15.114:6817/673244801","heartbeat_front_addr":"172.21.15.114:6815/673244801","state":["exists","up"]},{"osd":2,"uuid":"73a72ace-7f75-43c3-aab0-7ca73811182d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6819","nonce":4280964857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6821","nonce":4280964857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6825","nonce":4280964857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6823","nonce":4280964857}]},"public_addr":"172.21.15.114:6819/4280964857","cluster_addr":"172.21.15.114:6821/4280964857","heartbeat_back_addr":"172.21.15.114:6825/4280964857","heartbeat_front_addr":"172.21.15.114:6823/4280964857","state":["exists","up"]},{"osd":3,"uuid":"cf107b20-067f-407b-bb0e-0e09e4e03029","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6827","nonce":575349472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6829","nonce":575349472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6832","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6833","nonce":575349472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6831","nonce":575349472}]},"public_addr":"172.21.15.114:6827/575349472","cluster_addr":"172.21.15.114:6829/575349472","heartbeat_back_addr":"172.21.15.114:6833/575349472","heartbeat_front_addr":"172.21.15.114:6831/575349472","state":["exists","up"]},{"osd":4,"uuid":"d2921ec7-d207-4918-9f9e-5c81f71d12ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6801","nonce":3259223788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6803","nonce":3259223788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6807","nonce":3259223788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6805","nonce":3259223788}]},"public_addr":"172.21.15.121:6801/3259223788","cluster_addr":"172.21.15.121:6803/3259223788","heartbeat_back_addr":"172.21.15.121:6807/3259223788","heartbeat_front_addr":"172.21.15.121:6805/3259223788","state":["exists","up"]},{"osd":5,"uuid":"d136ab9a-1360-4cf1-a932-b57c1e59436a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6809","nonce":2151936015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6811","nonce":2151936015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6815","nonce":2151936015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6813","nonce":2151936015}]},"public_addr":"172.21.15.121:6809/2151936015","cluster_addr":"172.21.15.121:6811/2151936015","heartbeat_back_addr":"172.21.15.121:6815/2151936015","heartbeat_front_addr":"172.21.15.121:6813/2151936015","state":["exists","up"]},{"osd":6,"uuid":"d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6817","nonce":877204710}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6819","nonce":877204710}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6823","nonce":877204710}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6821","nonce":877204710}]},"public_addr":"172.21.15.121:6817/877204710","cluster_addr":"172.21.15.121:6819/877204710","heartbeat_back_addr":"172.21.15.121:6823/877204710","heartbeat_front_addr":"172.21.15.121:6821/877204710","state":["exists","up"]},{"osd":7,"uuid":"40188450-fd99-40b6-a68d-19e286af2201","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6825","nonce":3388831357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6827","nonce":3388831357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6831","nonce":3388831357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6829","nonce":3388831357}]},"public_addr":"172.21.15.121:6825/3388831357","cluster_addr":"172.21.15.121:6827/3388831357","heartbeat_back_addr":"172.21.15.121:6831/3388831357","heartbeat_front_addr":"172.21.15.121:6829/3388831357","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:48:31.081380+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:48:53.229307+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:49:16.706843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:49:38.208475+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:00.094075+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:26.145120+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:50.839135+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:51:15.262186+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.114:6801/4206988749":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/2424716870":"2024-09-14T21:47:14.453076+0000","172.21.15.114:6800/4206988749":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/3146194524":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/2980892939":"2024-09-14T21:46:52.559074+0000","172.21.15.114:6800/2918421085":"2024-09-14T21:46:52.559074+0000","172.21.15.114:6801/2918421085":"2024-09-14T21:46:52.559074+0000","172.21.15.114:0/26654585":"2024-09-14T21:46:52.559074+0000","172.21.15.114:0/206701471":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/115794601":"2024-09-14T21:46:52.559074+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T21:51:22.775 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:22 smithi114 ceph-mon[23730]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T21:51:22.775 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:22 smithi114 ceph-mon[23730]: pgmap v181: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:22.775 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:22 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3258453040' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:51:22.775 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:22 smithi114 ceph-mon[28528]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T21:51:22.775 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:22 smithi114 ceph-mon[28528]: pgmap v181: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:22.775 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:22 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3258453040' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:51:22.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:22 smithi121 ceph-mon[24681]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T21:51:22.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:22 smithi121 ceph-mon[24681]: pgmap v181: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:22.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:22 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3258453040' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:51:23.023 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T21:49:18.809652+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T21:51:23.024 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T21:51:23.225 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:51:24.834 INFO:teuthology.orchestra.run.smithi114.stdout:pg_num: 1 2024-09-13T21:51:25.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:24 smithi114 ceph-mon[23730]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-13T21:51:25.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:24 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2432543038' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T21:51:25.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:24 smithi114 ceph-mon[28528]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-13T21:51:25.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:24 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2432543038' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T21:51:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:24 smithi121 ceph-mon[24681]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-13T21:51:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:24 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2432543038' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T21:51:26.002 INFO:tasks.cephadm:Adding prometheus.a on smithi121 2024-09-13T21:51:26.002 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi121=a' 2024-09-13T21:51:26.559 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:26 smithi121 ceph-mon[24681]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-13T21:51:26.616 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:26 smithi114 ceph-mon[23730]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-13T21:51:26.616 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:26 smithi114 ceph-mon[28528]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-13T21:51:27.967 INFO:teuthology.orchestra.run.smithi121.stdout:Scheduled prometheus update... 2024-09-13T21:51:28.576 DEBUG:teuthology.orchestra.run.smithi121:prometheus.a> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T21:51:28.578 INFO:tasks.cephadm:Adding node-exporter.a on smithi114 2024-09-13T21:51:28.578 INFO:tasks.cephadm:Adding node-exporter.b on smithi121 2024-09-13T21:51:28.578 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi114=a;smithi121=b' 2024-09-13T21:51:29.222 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ignoring --setuser ceph since I am not root 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ignoring --setgroup ceph since I am not root 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-mgr[26213]: -- 172.21.15.121:0/2753432881 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564febb7a340 con 0x564febb53000 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:29.150+0000 7f5eab616000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: from='client.24350 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: Saving service prometheus spec with placement smithi121=a;count:1 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:29.223 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:29.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:29.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:29.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:29.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:28 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:51:29.232 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:51:29.232 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: from='client.24350 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:29.232 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: Saving service prometheus spec with placement smithi121=a;count:1 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setuser ceph since I am not root 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setgroup ceph since I am not root 2024-09-13T21:51:29.233 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-mgr[23963]: -- 172.21.15.114:0/2768361286 <== mon.1 v2:172.21.15.114:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564726972340 con 0x56472694b000 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:29.153+0000 7fb85575a000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:29.230+0000 7fb85575a000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: from='client.24350 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: Saving service prometheus spec with placement smithi121=a;count:1 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:29.234 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' 2024-09-13T21:51:29.235 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:28 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:51:29.493 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:29.404+0000 7fb85575a000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T21:51:29.493 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:29.229+0000 7f5eab616000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T21:51:29.493 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:29.402+0000 7f5eab616000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T21:51:29.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:29.491+0000 7fb85575a000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T21:51:29.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:29.623+0000 7fb85575a000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:51:29.836 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:29.491+0000 7f5eab616000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T21:51:29.836 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:29.624+0000 7f5eab616000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:51:30.104 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-mon[24681]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:51:30.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-mon[24681]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T21:51:30.105 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:29.834+0000 7f5eab616000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T21:51:30.163 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-mon[23730]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:51:30.163 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-mon[23730]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T21:51:30.163 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:29.833+0000 7fb85575a000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T21:51:30.164 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-mon[28528]: from='mgr.14152 172.21.15.114:0/2915315434' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:51:30.164 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:29 smithi114 ceph-mon[28528]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T21:51:30.355 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:30.174+0000 7f5eab616000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T21:51:30.355 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:30.264+0000 7f5eab616000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T21:51:30.415 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:30.161+0000 7fb85575a000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T21:51:30.415 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:30.249+0000 7fb85575a000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T21:51:30.415 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:30.413+0000 7fb85575a000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T21:51:30.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:30.438+0000 7f5eab616000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T21:51:31.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:31.458+0000 7fb85575a000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T21:51:31.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:31.548+0000 7fb85575a000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T21:51:31.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:31.487+0000 7f5eab616000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T21:51:31.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:31.576+0000 7f5eab616000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T21:51:32.751 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:32.440+0000 7fb85575a000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T21:51:32.772 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:32.465+0000 7f5eab616000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T21:51:33.071 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:32.749+0000 7fb85575a000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T21:51:33.071 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:32.882+0000 7fb85575a000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:51:33.071 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:32.971+0000 7fb85575a000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T21:51:33.071 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.069+0000 7fb85575a000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T21:51:33.089 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:32.769+0000 7f5eab616000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T21:51:33.089 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:32.903+0000 7f5eab616000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:51:33.089 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:32.989+0000 7f5eab616000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T21:51:33.090 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.087+0000 7f5eab616000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T21:51:33.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.158+0000 7fb85575a000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T21:51:33.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.306+0000 7fb85575a000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T21:51:33.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.175+0000 7f5eab616000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T21:51:33.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.322+0000 7f5eab616000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T21:51:33.781 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.400+0000 7fb85575a000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T21:51:33.782 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.496+0000 7fb85575a000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T21:51:33.795 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.415+0000 7f5eab616000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T21:51:33.795 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.511+0000 7f5eab616000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T21:51:34.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.780+0000 7fb85575a000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T21:51:34.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.869+0000 7fb85575a000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T21:51:34.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:33.958+0000 7fb85575a000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T21:51:34.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.792+0000 7f5eab616000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T21:51:34.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.880+0000 7f5eab616000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T21:51:34.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:33.969+0000 7f5eab616000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T21:51:35.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: Standby manager daemon x restarted 2024-09-13T21:51:35.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T21:51:35.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:51:35.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: Active manager daemon y restarted 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: Activating manager daemon y 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:35.026+0000 7fb85575a000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:51:35] ENGINE Bus STARTING 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: CherryPy Checker: 2024-09-13T21:51:35.299 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: The Application mounted at '' has an empty config. 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: Standby manager daemon x restarted 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: Active manager daemon y restarted 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: Activating manager daemon y 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T21:51:35.300 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: Standby manager daemon x restarted 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: Active manager daemon y restarted 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: Activating manager daemon y 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/740163755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:35.023+0000 7f5eab616000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T21:51:35.301 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: [13/Sep/2024:21:51:35] ENGINE Bus STARTING 2024-09-13T21:51:35.302 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: CherryPy Checker: 2024-09-13T21:51:35.302 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: The Application mounted at '' has an empty config. 2024-09-13T21:51:35.302 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: 2024-09-13T21:51:35.302 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: [13/Sep/2024:21:51:35] ENGINE Serving on http://:::9283 2024-09-13T21:51:35.302 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:51:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: [13/Sep/2024:21:51:35] ENGINE Bus STARTED 2024-09-13T21:51:35.553 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T21:51:35.554 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:51:35] ENGINE Serving on http://:::9283 2024-09-13T21:51:35.554 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:51:35] ENGINE Bus STARTED 2024-09-13T21:51:35.554 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:51:35] ENGINE Bus STARTING 2024-09-13T21:51:35.554 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:51:35] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:51:35.554 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:21:51:35] ENGINE Bus STARTED 2024-09-13T21:51:36.115 INFO:teuthology.orchestra.run.smithi121.stdout:Scheduled node-exporter update... 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: mgrmap e17: y(active, starting, since 0.0543473s), standbys: x 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:51:36.365 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: Manager daemon y is now available 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:36.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]': finished 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]': finished 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: [13/Sep/2024:21:51:35] ENGINE Bus STARTING 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: [13/Sep/2024:21:51:35] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: [13/Sep/2024:21:51:35] ENGINE Bus STARTED 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:51:36.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:51:36.368 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.368 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.368 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.368 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: mgrmap e17: y(active, starting, since 0.0543473s), standbys: x 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:51:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: Manager daemon y is now available 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]': finished 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: mgrmap e17: y(active, starting, since 0.0543473s), standbys: x 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T21:51:36.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:51:36.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: Manager daemon y is now available 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]': finished 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T21:51:36.579 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]': finished 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: [13/Sep/2024:21:51:35] ENGINE Bus STARTING 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: [13/Sep/2024:21:51:35] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: [13/Sep/2024:21:51:35] ENGINE Bus STARTED 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:51:36.580 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]': finished 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: [13/Sep/2024:21:51:35] ENGINE Bus STARTING 2024-09-13T21:51:36.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: [13/Sep/2024:21:51:35] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: [13/Sep/2024:21:51:35] ENGINE Bus STARTED 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:36.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:51:37.025 DEBUG:teuthology.orchestra.run.smithi114:node-exporter.a> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T21:51:37.028 DEBUG:teuthology.orchestra.run.smithi121:node-exporter.b> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T21:51:37.030 INFO:tasks.cephadm:Adding alertmanager.a on smithi114 2024-09-13T21:51:37.031 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi114=a' 2024-09-13T21:51:37.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:37 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:51:37.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:37 smithi121 ceph-mon[24681]: mgrmap e18: y(active, since 1.06645s), standbys: x 2024-09-13T21:51:37.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:37 smithi121 ceph-mon[24681]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi114=a;smithi121=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:37.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:37 smithi121 ceph-mon[24681]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:37.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:37 smithi121 ceph-mon[24681]: Saving service node-exporter spec with placement smithi114=a;smithi121=b;count:2 2024-09-13T21:51:37.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:37 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:37.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:37 smithi121 ceph-mon[24681]: Deploying daemon prometheus.a on smithi121 2024-09-13T21:51:37.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[28528]: mgrmap e18: y(active, since 1.06645s), standbys: x 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[28528]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi114=a;smithi121=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[28528]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[28528]: Saving service node-exporter spec with placement smithi114=a;smithi121=b;count:2 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[28528]: Deploying daemon prometheus.a on smithi121 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[23730]: mgrmap e18: y(active, since 1.06645s), standbys: x 2024-09-13T21:51:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[23730]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi114=a;smithi121=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:37.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[23730]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:37.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[23730]: Saving service node-exporter spec with placement smithi114=a;smithi121=b;count:2 2024-09-13T21:51:37.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:37.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:37 smithi114 ceph-mon[23730]: Deploying daemon prometheus.a on smithi121 2024-09-13T21:51:38.306 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:38 smithi121 ceph-mon[24681]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:38.306 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:38 smithi121 ceph-mon[24681]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T21:51:38.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:38 smithi114 ceph-mon[23730]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:38.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:38 smithi114 ceph-mon[23730]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T21:51:38.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:38 smithi114 ceph-mon[28528]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:38.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:38 smithi114 ceph-mon[28528]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T21:51:40.425 INFO:teuthology.orchestra.run.smithi121.stdout:Scheduled alertmanager update... 2024-09-13T21:51:40.426 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:40 smithi121 ceph-mon[24681]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:40.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:40 smithi114 ceph-mon[23730]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:40.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:40 smithi114 ceph-mon[28528]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:40.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:51:40] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:51:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:41 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:41 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:41 smithi121 ceph-mon[24681]: from='client.24398 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi114=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:41 smithi121 ceph-mon[24681]: Saving service alertmanager spec with placement smithi114=a;count:1 2024-09-13T21:51:41.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:41 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.518 DEBUG:teuthology.orchestra.run.smithi114:alertmanager.a> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T21:51:41.521 INFO:tasks.cephadm:Adding grafana.a on smithi121 2024-09-13T21:51:41.521 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi121=a' 2024-09-13T21:51:41.546 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.546 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.546 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[28528]: from='client.24398 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi114=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:41.546 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[28528]: Saving service alertmanager spec with placement smithi114=a;count:1 2024-09-13T21:51:41.546 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.546 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.546 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:41.547 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[23730]: from='client.24398 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi114=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:41.547 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[23730]: Saving service alertmanager spec with placement smithi114=a;count:1 2024-09-13T21:51:41.547 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:41 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:42.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:42 smithi114 ceph-mon[23730]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:42.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:42 smithi114 ceph-mon[28528]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:42.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:42 smithi121 ceph-mon[24681]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:44.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:44 smithi114 ceph-mon[23730]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:44 smithi114 ceph-mon[28528]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:44 smithi121 ceph-mon[24681]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:45.890 INFO:teuthology.orchestra.run.smithi121.stdout:Scheduled grafana update... 2024-09-13T21:51:46.715 DEBUG:teuthology.orchestra.run.smithi121:grafana.a> sudo journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T21:51:46.718 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T21:51:46.718 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T21:51:47.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:46 smithi121 ceph-mon[24681]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:47.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:46 smithi121 ceph-mon[24681]: from='client.24404 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:47.005 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:46 smithi121 ceph-mon[24681]: Saving service grafana spec with placement smithi121=a;count:1 2024-09-13T21:51:47.005 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[23730]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[23730]: from='client.24404 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[23730]: Saving service grafana spec with placement smithi121=a;count:1 2024-09-13T21:51:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:47.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[28528]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:47.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[28528]: from='client.24404 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:47.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[28528]: Saving service grafana spec with placement smithi121=a;count:1 2024-09-13T21:51:47.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:48 smithi114 ceph-mon[23730]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:49.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:48 smithi114 ceph-mon[28528]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:48 smithi121 ceph-mon[24681]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:49.999 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:49 smithi121 systemd[1]: Starting Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:51:50.043 INFO:teuthology.orchestra.run.smithi114.stdout:[client.0] 2024-09-13T21:51:50.043 INFO:teuthology.orchestra.run.smithi114.stdout: key = AQD2s+RmtPFLAhAAaUACId4anMjhlRGbvrDfeQ== 2024-09-13T21:51:50.341 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 podman[40936]: 2024-09-13 21:51:50.110863507 +0000 UTC m=+0.016721351 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-09-13T21:51:50.342 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 podman[40936]: 2024-09-13 21:51:50.230849844 +0000 UTC m=+0.136707678 container create 05dcec8c97fe3e72edd1e61c9f15283d98676233aef36e14e018e512bda677cb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:51:50.621 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 podman[40936]: 2024-09-13 21:51:50.474792746 +0000 UTC m=+0.380650581 container init 05dcec8c97fe3e72edd1e61c9f15283d98676233aef36e14e018e512bda677cb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:51:50.621 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 podman[40936]: 2024-09-13 21:51:50.477661665 +0000 UTC m=+0.383519502 container start 05dcec8c97fe3e72edd1e61c9f15283d98676233aef36e14e018e512bda677cb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:51:50.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.501Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-13T21:51:50.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.501Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-09-13T21:51:50.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.501Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-09-13T21:51:50.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.501Z caller=main.go:518 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi121 (none))" 2024-09-13T21:51:50.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.501Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T21:51:50.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.501Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.502Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.503Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.504Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.506Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.506Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.355µs 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.506Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.506Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-13T21:51:50.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.506Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32µs wal_replay_duration=285.786µs total_replay_duration=340.751µs 2024-09-13T21:51:50.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.507Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T21:51:50.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.507Z caller=main.go:947 level=info msg="TSDB started" 2024-09-13T21:51:50.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.507Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T21:51:50.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.521Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.563393ms db_storage=634ns remote_storage=1.636µs web_handler=282ns query_engine=527ns scrape=431.159µs scrape_sd=15.516µs notify=697ns notify_sd=1.245µs rules=13.948932ms 2024-09-13T21:51:50.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:51:50.521Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-09-13T21:51:50.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 bash[40936]: 05dcec8c97fe3e72edd1e61c9f15283d98676233aef36e14e018e512bda677cb 2024-09-13T21:51:50.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:51:50 smithi121 systemd[1]: Started Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:51:50.647 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-09-13T21:51:50.647 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T21:51:50.647 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T21:51:50.687 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2715299408' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2715299408' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.121:9095"}]: dispatch 2024-09-13T21:51:50.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:50.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2715299408' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2715299408' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.121:9095"}]: dispatch 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2715299408' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2715299408' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:51:51.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.121:9095"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:51.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:51.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:51.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:51.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:51 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:51.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:51 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:51:51.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:51 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.121:9095"}]: dispatch 2024-09-13T21:51:51.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:51 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:51.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:51 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.121:9095"}]: dispatch 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.121:9095"}]: dispatch 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:51 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:51:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:52 smithi114 ceph-mon[23730]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:52 smithi114 ceph-mon[28528]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:52 smithi121 ceph-mon[24681]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:54.728 INFO:teuthology.orchestra.run.smithi121.stdout:[client.1] 2024-09-13T21:51:54.728 INFO:teuthology.orchestra.run.smithi121.stdout: key = AQD6s+Rmrb4GKxAAcWeOeja0hrQfQsDJ6p65Jg== 2024-09-13T21:51:54.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:54 smithi121 ceph-mon[24681]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:54.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:54 smithi121 ceph-mon[24681]: from='client.? 172.21.15.121:0/1600476676' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:54.869 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:54 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:54.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:54 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:51:55.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[23730]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:55.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[23730]: from='client.? 172.21.15.121:0/1600476676' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:55.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:55.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:51:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[28528]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[28528]: from='client.? 172.21.15.121:0/1600476676' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:51:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:54 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:51:55.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:51:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:51:56.353 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-09-13T21:51:56.353 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T21:51:56.353 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T21:51:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:56 smithi121 ceph-mon[24681]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:56.393 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T21:51:56.393 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T21:51:56.394 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T21:51:56.420 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:56 smithi114 ceph-mon[23730]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:56.420 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:56.420 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:56.420 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:56 smithi114 ceph-mon[28528]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:56.420 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:56.420 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:56.606 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:51:58.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:51:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:51:58.235 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:51:58.925 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":19,"active_gid":24346,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":209115709},{"type":"v1","addr":"172.21.15.114:6801","nonce":209115709}]},"active_addr":"172.21.15.114:6801/209115709","active_change":"2024-09-13T21:51:35.030182+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24356,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.114:8443/","prometheus":"http://172.21.15.114:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.114:0","nonce":3559360226}]},{"addrvec":[{"type":"v2","addr":"172.21.15.114:0","nonce":3853349338}]},{"addrvec":[{"type":"v2","addr":"172.21.15.114:0","nonce":2273874582}]},{"addrvec":[{"type":"v2","addr":"172.21.15.114:0","nonce":2622975080}]}]}} 2024-09-13T21:51:58.928 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T21:51:58.928 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T21:51:58.928 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T21:51:59.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:58 smithi121 ceph-mon[24681]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:59.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:58 smithi121 ceph-mon[24681]: Detected new or changed devices on smithi121 2024-09-13T21:51:59.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:59.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:51:58 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2572318404' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:51:59.170 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:51:59.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[28528]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:59.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[28528]: Detected new or changed devices on smithi121 2024-09-13T21:51:59.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:59.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2572318404' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:51:59.291 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[23730]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:51:59.291 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[23730]: Detected new or changed devices on smithi121 2024-09-13T21:51:59.292 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:51:59.292 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:51:58 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2572318404' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:51:59.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:51:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:51:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:00.476 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:52:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:00.728 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: Deploying daemon node-exporter.a on smithi114 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.729 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: Deploying daemon node-exporter.a on smithi114 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.730 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:00.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:00.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:00.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: Deploying daemon node-exporter.a on smithi114 2024-09-13T21:52:00.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:00.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:01.047 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:52:01.047 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":59,"fsid":"8ccc9cd8-7219-11ef-bceb-c7b262605968","created":"2024-09-13T21:46:19.540411+0000","modified":"2024-09-13T21:51:35.029780+0000","last_up_change":"2024-09-13T21:51:17.202594+0000","last_in_change":"2024-09-13T21:50:58.692065+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T21:49:18.809652+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"bfe0a368-759e-4b6e-b492-b83b86bd0883","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6803","nonce":965608804}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6805","nonce":965608804}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6809","nonce":965608804}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6807","nonce":965608804}]},"public_addr":"172.21.15.114:6803/965608804","cluster_addr":"172.21.15.114:6805/965608804","heartbeat_back_addr":"172.21.15.114:6809/965608804","heartbeat_front_addr":"172.21.15.114:6807/965608804","state":["exists","up"]},{"osd":1,"uuid":"b13c8473-564d-4977-9d35-5e7eccb2320f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6811","nonce":673244801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6813","nonce":673244801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6817","nonce":673244801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6815","nonce":673244801}]},"public_addr":"172.21.15.114:6811/673244801","cluster_addr":"172.21.15.114:6813/673244801","heartbeat_back_addr":"172.21.15.114:6817/673244801","heartbeat_front_addr":"172.21.15.114:6815/673244801","state":["exists","up"]},{"osd":2,"uuid":"73a72ace-7f75-43c3-aab0-7ca73811182d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6819","nonce":4280964857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6821","nonce":4280964857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6825","nonce":4280964857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6823","nonce":4280964857}]},"public_addr":"172.21.15.114:6819/4280964857","cluster_addr":"172.21.15.114:6821/4280964857","heartbeat_back_addr":"172.21.15.114:6825/4280964857","heartbeat_front_addr":"172.21.15.114:6823/4280964857","state":["exists","up"]},{"osd":3,"uuid":"cf107b20-067f-407b-bb0e-0e09e4e03029","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6827","nonce":575349472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6829","nonce":575349472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6832","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6833","nonce":575349472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6831","nonce":575349472}]},"public_addr":"172.21.15.114:6827/575349472","cluster_addr":"172.21.15.114:6829/575349472","heartbeat_back_addr":"172.21.15.114:6833/575349472","heartbeat_front_addr":"172.21.15.114:6831/575349472","state":["exists","up"]},{"osd":4,"uuid":"d2921ec7-d207-4918-9f9e-5c81f71d12ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6801","nonce":3259223788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6803","nonce":3259223788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6807","nonce":3259223788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6805","nonce":3259223788}]},"public_addr":"172.21.15.121:6801/3259223788","cluster_addr":"172.21.15.121:6803/3259223788","heartbeat_back_addr":"172.21.15.121:6807/3259223788","heartbeat_front_addr":"172.21.15.121:6805/3259223788","state":["exists","up"]},{"osd":5,"uuid":"d136ab9a-1360-4cf1-a932-b57c1e59436a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6809","nonce":2151936015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6811","nonce":2151936015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6815","nonce":2151936015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6813","nonce":2151936015}]},"public_addr":"172.21.15.121:6809/2151936015","cluster_addr":"172.21.15.121:6811/2151936015","heartbeat_back_addr":"172.21.15.121:6815/2151936015","heartbeat_front_addr":"172.21.15.121:6813/2151936015","state":["exists","up"]},{"osd":6,"uuid":"d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6817","nonce":877204710}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6819","nonce":877204710}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6823","nonce":877204710}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6821","nonce":877204710}]},"public_addr":"172.21.15.121:6817/877204710","cluster_addr":"172.21.15.121:6819/877204710","heartbeat_back_addr":"172.21.15.121:6823/877204710","heartbeat_front_addr":"172.21.15.121:6821/877204710","state":["exists","up"]},{"osd":7,"uuid":"40188450-fd99-40b6-a68d-19e286af2201","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6825","nonce":3388831357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6827","nonce":3388831357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6831","nonce":3388831357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6829","nonce":3388831357}]},"public_addr":"172.21.15.121:6825/3388831357","cluster_addr":"172.21.15.121:6827/3388831357","heartbeat_back_addr":"172.21.15.121:6831/3388831357","heartbeat_front_addr":"172.21.15.121:6829/3388831357","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:48:31.081380+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:48:53.229307+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:49:16.706843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:49:38.208475+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:00.094075+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:26.145120+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:50.839135+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:51:15.262186+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.114:0/3483901434":"2024-09-14T21:51:35.029733+0000","172.21.15.114:0/4155242912":"2024-09-14T21:51:35.029733+0000","172.21.15.114:0/1462675355":"2024-09-14T21:51:35.029733+0000","172.21.15.114:6801/408371097":"2024-09-14T21:51:35.029733+0000","172.21.15.114:6800/408371097":"2024-09-14T21:51:35.029733+0000","172.21.15.114:0/1573161012":"2024-09-14T21:51:35.029733+0000","172.21.15.114:6801/4206988749":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/2424716870":"2024-09-14T21:47:14.453076+0000","172.21.15.114:6800/4206988749":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/3146194524":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/2980892939":"2024-09-14T21:46:52.559074+0000","172.21.15.114:6800/2918421085":"2024-09-14T21:46:52.559074+0000","172.21.15.114:6801/2918421085":"2024-09-14T21:46:52.559074+0000","172.21.15.114:0/26654585":"2024-09-14T21:46:52.559074+0000","172.21.15.114:0/206701471":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/115794601":"2024-09-14T21:46:52.559074+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T21:52:01.048 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:00 smithi114 bash[44699]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-09-13T21:52:01.570 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:01 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3841976568' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:52:01.571 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:01 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3841976568' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:52:01.571 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:01 smithi114 bash[44699]: Getting image source signatures 2024-09-13T21:52:01.571 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:01 smithi114 bash[44699]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-09-13T21:52:01.571 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:01 smithi114 bash[44699]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-09-13T21:52:01.571 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:01 smithi114 bash[44699]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-09-13T21:52:01.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:01 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3841976568' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:52:02.762 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:02 smithi114 ceph-mon[23730]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:02.762 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:02 smithi114 ceph-mon[28528]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:02.782 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T21:52:02.782 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T21:52:02.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:02 smithi121 ceph-mon[24681]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:03.007 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:03.324 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:03 smithi114 bash[44699]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-09-13T21:52:03.325 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:03 smithi114 bash[44699]: Writing manifest to image destination 2024-09-13T21:52:03.969 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:03 smithi114 podman[44699]: 2024-09-13 21:52:03.74558029 +0000 UTC m=+3.129921278 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-09-13T21:52:03.969 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:03 smithi114 podman[44699]: 2024-09-13 21:52:03.96892122 +0000 UTC m=+3.353262214 container create 45aa09d329e6e03fdf9727f6a424e56446fcfe68f5c614f6c02946ea1bb0642e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 podman[44699]: 2024-09-13 21:52:04.179455804 +0000 UTC m=+3.563796802 container init 45aa09d329e6e03fdf9727f6a424e56446fcfe68f5c614f6c02946ea1bb0642e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 podman[44699]: 2024-09-13 21:52:04.182175564 +0000 UTC m=+3.566516556 container start 45aa09d329e6e03fdf9727f6a424e56446fcfe68f5c614f6c02946ea1bb0642e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.185Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.185Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.186Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.186Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=arp 2024-09-13T21:52:04.264 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-13T21:52:04.265 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=edac 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-13T21:52:04.266 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=os 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=stat 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-13T21:52:04.267 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-13T21:52:04.268 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-13T21:52:04.268 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=time 2024-09-13T21:52:04.268 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-13T21:52:04.268 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=uname 2024-09-13T21:52:04.268 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-13T21:52:04.268 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-13T21:52:04.268 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-13T21:52:04.269 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-13T21:52:04.269 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[45048]: ts=2024-09-13T21:52:04.187Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-13T21:52:04.269 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 bash[44699]: 45aa09d329e6e03fdf9727f6a424e56446fcfe68f5c614f6c02946ea1bb0642e 2024-09-13T21:52:04.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-mon[28528]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:04.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:04.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 21:52:04 smithi114 systemd[1]: Started Ceph node-exporter.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:52:04.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-mon[23730]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:04.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:04 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:04 smithi121 ceph-mon[24681]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:04 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:05.173 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:04 smithi121 systemd[1]: Starting Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:52:05.427 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:52:05.427 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":59,"fsid":"8ccc9cd8-7219-11ef-bceb-c7b262605968","created":"2024-09-13T21:46:19.540411+0000","modified":"2024-09-13T21:51:35.029780+0000","last_up_change":"2024-09-13T21:51:17.202594+0000","last_in_change":"2024-09-13T21:50:58.692065+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T21:49:18.809652+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"bfe0a368-759e-4b6e-b492-b83b86bd0883","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6803","nonce":965608804}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6805","nonce":965608804}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6808","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6809","nonce":965608804}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":965608804},{"type":"v1","addr":"172.21.15.114:6807","nonce":965608804}]},"public_addr":"172.21.15.114:6803/965608804","cluster_addr":"172.21.15.114:6805/965608804","heartbeat_back_addr":"172.21.15.114:6809/965608804","heartbeat_front_addr":"172.21.15.114:6807/965608804","state":["exists","up"]},{"osd":1,"uuid":"b13c8473-564d-4977-9d35-5e7eccb2320f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6810","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6811","nonce":673244801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6812","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6813","nonce":673244801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6816","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6817","nonce":673244801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6814","nonce":673244801},{"type":"v1","addr":"172.21.15.114:6815","nonce":673244801}]},"public_addr":"172.21.15.114:6811/673244801","cluster_addr":"172.21.15.114:6813/673244801","heartbeat_back_addr":"172.21.15.114:6817/673244801","heartbeat_front_addr":"172.21.15.114:6815/673244801","state":["exists","up"]},{"osd":2,"uuid":"73a72ace-7f75-43c3-aab0-7ca73811182d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6818","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6819","nonce":4280964857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6820","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6821","nonce":4280964857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6824","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6825","nonce":4280964857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6822","nonce":4280964857},{"type":"v1","addr":"172.21.15.114:6823","nonce":4280964857}]},"public_addr":"172.21.15.114:6819/4280964857","cluster_addr":"172.21.15.114:6821/4280964857","heartbeat_back_addr":"172.21.15.114:6825/4280964857","heartbeat_front_addr":"172.21.15.114:6823/4280964857","state":["exists","up"]},{"osd":3,"uuid":"cf107b20-067f-407b-bb0e-0e09e4e03029","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6826","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6827","nonce":575349472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6828","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6829","nonce":575349472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6832","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6833","nonce":575349472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6830","nonce":575349472},{"type":"v1","addr":"172.21.15.114:6831","nonce":575349472}]},"public_addr":"172.21.15.114:6827/575349472","cluster_addr":"172.21.15.114:6829/575349472","heartbeat_back_addr":"172.21.15.114:6833/575349472","heartbeat_front_addr":"172.21.15.114:6831/575349472","state":["exists","up"]},{"osd":4,"uuid":"d2921ec7-d207-4918-9f9e-5c81f71d12ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6801","nonce":3259223788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6803","nonce":3259223788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6807","nonce":3259223788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":3259223788},{"type":"v1","addr":"172.21.15.121:6805","nonce":3259223788}]},"public_addr":"172.21.15.121:6801/3259223788","cluster_addr":"172.21.15.121:6803/3259223788","heartbeat_back_addr":"172.21.15.121:6807/3259223788","heartbeat_front_addr":"172.21.15.121:6805/3259223788","state":["exists","up"]},{"osd":5,"uuid":"d136ab9a-1360-4cf1-a932-b57c1e59436a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6809","nonce":2151936015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6811","nonce":2151936015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6815","nonce":2151936015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":2151936015},{"type":"v1","addr":"172.21.15.121:6813","nonce":2151936015}]},"public_addr":"172.21.15.121:6809/2151936015","cluster_addr":"172.21.15.121:6811/2151936015","heartbeat_back_addr":"172.21.15.121:6815/2151936015","heartbeat_front_addr":"172.21.15.121:6813/2151936015","state":["exists","up"]},{"osd":6,"uuid":"d1f3d13c-2fb0-4338-9d3b-fa1b0de8208c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6817","nonce":877204710}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6819","nonce":877204710}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6823","nonce":877204710}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":877204710},{"type":"v1","addr":"172.21.15.121:6821","nonce":877204710}]},"public_addr":"172.21.15.121:6817/877204710","cluster_addr":"172.21.15.121:6819/877204710","heartbeat_back_addr":"172.21.15.121:6823/877204710","heartbeat_front_addr":"172.21.15.121:6821/877204710","state":["exists","up"]},{"osd":7,"uuid":"40188450-fd99-40b6-a68d-19e286af2201","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6825","nonce":3388831357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6827","nonce":3388831357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6831","nonce":3388831357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3388831357},{"type":"v1","addr":"172.21.15.121:6829","nonce":3388831357}]},"public_addr":"172.21.15.121:6825/3388831357","cluster_addr":"172.21.15.121:6827/3388831357","heartbeat_back_addr":"172.21.15.121:6831/3388831357","heartbeat_front_addr":"172.21.15.121:6829/3388831357","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:48:31.081380+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:48:53.229307+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:49:16.706843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:49:38.208475+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:00.094075+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:26.145120+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:50:50.839135+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:51:15.262186+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.114:0/3483901434":"2024-09-14T21:51:35.029733+0000","172.21.15.114:0/4155242912":"2024-09-14T21:51:35.029733+0000","172.21.15.114:0/1462675355":"2024-09-14T21:51:35.029733+0000","172.21.15.114:6801/408371097":"2024-09-14T21:51:35.029733+0000","172.21.15.114:6800/408371097":"2024-09-14T21:51:35.029733+0000","172.21.15.114:0/1573161012":"2024-09-14T21:51:35.029733+0000","172.21.15.114:6801/4206988749":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/2424716870":"2024-09-14T21:47:14.453076+0000","172.21.15.114:6800/4206988749":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/3146194524":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/2980892939":"2024-09-14T21:46:52.559074+0000","172.21.15.114:6800/2918421085":"2024-09-14T21:46:52.559074+0000","172.21.15.114:6801/2918421085":"2024-09-14T21:46:52.559074+0000","172.21.15.114:0/26654585":"2024-09-14T21:46:52.559074+0000","172.21.15.114:0/206701471":"2024-09-14T21:47:14.453076+0000","172.21.15.114:0/115794601":"2024-09-14T21:46:52.559074+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T21:52:05.505 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:05 smithi121 bash[41914]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-09-13T21:52:05.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:05 smithi114 ceph-mon[23730]: Deploying daemon node-exporter.b on smithi121 2024-09-13T21:52:05.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:05 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2236708692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:52:05.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:05 smithi114 ceph-mon[28528]: Deploying daemon node-exporter.b on smithi121 2024-09-13T21:52:05.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:05 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2236708692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:52:05.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:05 smithi121 ceph-mon[24681]: Deploying daemon node-exporter.b on smithi121 2024-09-13T21:52:05.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:05 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2236708692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:52:05.871 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:05 smithi121 bash[41914]: Getting image source signatures 2024-09-13T21:52:05.872 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:05 smithi121 bash[41914]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-09-13T21:52:05.872 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:05 smithi121 bash[41914]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-09-13T21:52:05.872 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:05 smithi121 bash[41914]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-09-13T21:52:06.587 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:06 smithi114 ceph-mon[23730]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:06.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:06 smithi114 ceph-mon[28528]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:06.850 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:06 smithi121 ceph-mon[24681]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:07.012 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:52:06] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:07.041 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T21:52:07.041 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T21:52:07.042 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T21:52:07.042 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T21:52:07.043 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T21:52:07.043 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T21:52:07.044 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T21:52:07.044 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T21:52:07.121 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:06 smithi121 bash[41914]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-09-13T21:52:07.121 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 bash[41914]: Writing manifest to image destination 2024-09-13T21:52:07.583 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 podman[41914]: 2024-09-13 21:52:07.288866076 +0000 UTC m=+2.208817226 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-09-13T21:52:07.583 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 podman[41914]: 2024-09-13 21:52:07.387350892 +0000 UTC m=+2.307302075 container create 5452c9d8e4cb3f02ad4a05036d530edf2b62b65ff2cb58bbbf234a55516b15bb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T21:52:07.844 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:07 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:07.845 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:07 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:07.845 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:07 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:07.845 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:07 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:07.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:07.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:07.872 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 podman[41914]: 2024-09-13 21:52:07.58906251 +0000 UTC m=+2.509013647 container init 5452c9d8e4cb3f02ad4a05036d530edf2b62b65ff2cb58bbbf234a55516b15bb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T21:52:07.873 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 podman[41914]: 2024-09-13 21:52:07.591969506 +0000 UTC m=+2.511920653 container start 5452c9d8e4cb3f02ad4a05036d530edf2b62b65ff2cb58bbbf234a55516b15bb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T21:52:07.873 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.595Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-13T21:52:07.873 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.595Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-13T21:52:07.873 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.596Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-13T21:52:07.873 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.596Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T21:52:07.873 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-13T21:52:07.874 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=arp 2024-09-13T21:52:07.874 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-13T21:52:07.874 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-13T21:52:07.874 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-13T21:52:07.874 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-13T21:52:07.874 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-13T21:52:07.875 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-13T21:52:07.875 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-13T21:52:07.875 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-13T21:52:07.875 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=edac 2024-09-13T21:52:07.875 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-13T21:52:07.875 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-13T21:52:07.875 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-13T21:52:07.876 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-13T21:52:07.876 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-13T21:52:07.876 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-13T21:52:07.876 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-13T21:52:07.876 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-13T21:52:07.876 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-13T21:52:07.877 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-13T21:52:07.877 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-13T21:52:07.877 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-13T21:52:07.877 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-13T21:52:07.877 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-13T21:52:07.878 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-13T21:52:07.878 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-13T21:52:07.878 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=os 2024-09-13T21:52:07.878 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-13T21:52:07.878 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-13T21:52:07.879 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-13T21:52:07.879 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-13T21:52:07.879 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-13T21:52:07.879 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-13T21:52:07.879 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=stat 2024-09-13T21:52:07.879 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-13T21:52:07.880 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-13T21:52:07.880 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-13T21:52:07.880 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=time 2024-09-13T21:52:07.880 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-13T21:52:07.880 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=uname 2024-09-13T21:52:07.880 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-13T21:52:07.881 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-13T21:52:07.881 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-13T21:52:07.881 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-13T21:52:07.881 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[41988]: ts=2024-09-13T21:52:07.597Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-13T21:52:07.881 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 bash[41914]: 5452c9d8e4cb3f02ad4a05036d530edf2b62b65ff2cb58bbbf234a55516b15bb 2024-09-13T21:52:07.881 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 21:52:07 smithi121 systemd[1]: Started Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:52:08.142 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:08 smithi114 ceph-mon[23730]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:08 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:08 smithi114 ceph-mon[23730]: Deploying daemon alertmanager.a on smithi114 2024-09-13T21:52:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:08 smithi114 ceph-mon[28528]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:08 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:08 smithi114 ceph-mon[28528]: Deploying daemon alertmanager.a on smithi114 2024-09-13T21:52:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:08 smithi121 ceph-mon[24681]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:08 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:08 smithi121 ceph-mon[24681]: Deploying daemon alertmanager.a on smithi114 2024-09-13T21:52:10.124 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:11.012 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:10 smithi121 ceph-mon[24681]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:11.012 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:10 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:11.066 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:10 smithi114 ceph-mon[23730]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:11.066 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:10 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:11.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:10 smithi114 ceph-mon[28528]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:11.067 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:10 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:11.545 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:11.546 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:11.546 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:11.546 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:11.557 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:11.557 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:11.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:52:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:12 smithi121 ceph-mon[24681]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:13.062 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[23730]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:13.062 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:13.062 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:13.062 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:13.063 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:52:12] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:13.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[28528]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:13.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:13.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:13.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:13.510 INFO:teuthology.orchestra.run.smithi114.stdout:197568495634 2024-09-13T21:52:13.510 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T21:52:14.761 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:14 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:52:14] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:15.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:14 smithi114 ceph-mon[28528]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:15.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:14 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:15.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:14 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:15.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:14 smithi114 ceph-mon[23730]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:15.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:14 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:15.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:14 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:14 smithi121 ceph-mon[24681]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:15.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:14 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:15.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:14 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:16 smithi114 ceph-mon[23730]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:17.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:16 smithi114 ceph-mon[28528]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:16 smithi121 ceph-mon[24681]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:17.491 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:18 smithi114 ceph-mon[23730]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:18 smithi114 ceph-mon[28528]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:18 smithi121 ceph-mon[24681]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:20 smithi114 ceph-mon[23730]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:21.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:20 smithi114 ceph-mon[28528]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:20 smithi121 ceph-mon[24681]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:22.788 INFO:teuthology.orchestra.run.smithi114.stdout:77309411366 2024-09-13T21:52:22.788 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T21:52:23.070 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:22 smithi114 ceph-mon[23730]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:23.071 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:22 smithi114 ceph-mon[28528]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:22 smithi121 ceph-mon[24681]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:23.706 INFO:teuthology.orchestra.run.smithi114.stdout:55834574892 2024-09-13T21:52:23.707 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T21:52:25.001 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:24 smithi121 ceph-mon[24681]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:25.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:24 smithi114 ceph-mon[23730]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:25.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:24 smithi114 ceph-mon[28528]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:26.971 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:26 smithi114 ceph-mon[23730]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:26.971 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:26 smithi114 ceph-mon[28528]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:26.985 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:26 smithi121 ceph-mon[24681]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:26.994 INFO:teuthology.orchestra.run.smithi114.stdout:34359738416 2024-09-13T21:52:26.994 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T21:52:28.218 INFO:teuthology.orchestra.run.smithi114.stdout:231928234000 2024-09-13T21:52:28.218 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T21:52:28.964 INFO:teuthology.orchestra.run.smithi114.stdout:107374182434 2024-09-13T21:52:28.965 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T21:52:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:28 smithi114 ceph-mon[23730]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:28 smithi114 ceph-mon[28528]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:29.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:28 smithi121 ceph-mon[24681]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:30.793 INFO:teuthology.orchestra.run.smithi114.stdout:128849018911 2024-09-13T21:52:30.793 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T21:52:30.823 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:30.823 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:30.824 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:30.939 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:30 smithi114 ceph-mon[23730]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:30.939 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:30 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:30.939 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:30 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:30.940 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:30 smithi114 ceph-mon[28528]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:30.940 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:30 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:30.940 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:30 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:30.940 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:52:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:30 smithi121 ceph-mon[24681]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:31.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:30 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:31.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:30 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:31.431 INFO:teuthology.orchestra.run.smithi114.stdout:197568495637 2024-09-13T21:52:32.073 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:31 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2658251055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T21:52:32.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:31 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2658251055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T21:52:32.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:52:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:31 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2658251055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T21:52:33.020 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:32 smithi114 ceph-mon[23730]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:33.020 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:32 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:33.020 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:32 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:33.021 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:32 smithi114 ceph-mon[28528]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:33.021 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:32 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:33.021 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:32 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:32 smithi121 ceph-mon[24681]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:32 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:32 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:34.359 INFO:teuthology.orchestra.run.smithi114.stdout:163208757275 2024-09-13T21:52:34.360 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T21:52:35.160 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:34 smithi114 ceph-mon[23730]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:35.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:34 smithi114 ceph-mon[28528]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:35.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:34 smithi121 ceph-mon[24681]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:35.593 INFO:teuthology.orchestra.run.smithi114.stdout:34359738418 2024-09-13T21:52:35.895 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495634 got 197568495637 for osd.6 2024-09-13T21:52:35.895 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:36.156 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:52:36.156 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:52:36.156 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:35 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3364078196' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T21:52:36.156 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:52:36.156 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:52:36.156 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:35 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3364078196' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T21:52:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:52:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:52:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:35 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3364078196' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T21:52:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:36 smithi114 ceph-mon[23730]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:37.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:36 smithi114 ceph-mon[28528]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:37.135 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:37.136 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:36 smithi121 ceph-mon[24681]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:39.009 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:39.104 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:38 smithi114 ceph-mon[23730]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:39.105 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:38 smithi114 ceph-mon[28528]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:38 smithi121 ceph-mon[24681]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:39.525 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738418 for osd.0 2024-09-13T21:52:39.525 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:40.562 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:41.208 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:40 smithi114 ceph-mon[23730]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:41.208 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:40 smithi114 ceph-mon[28528]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:40 smithi121 ceph-mon[24681]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:42.558 INFO:teuthology.orchestra.run.smithi114.stdout:77309411371 2024-09-13T21:52:43.267 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:42 smithi114 ceph-mon[23730]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:43.268 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:42 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1626903793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T21:52:43.268 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:42 smithi114 ceph-mon[28528]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:43.268 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:42 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1626903793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T21:52:43.361 INFO:teuthology.orchestra.run.smithi114.stdout:128849018914 2024-09-13T21:52:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:42 smithi121 ceph-mon[24681]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:42 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1626903793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T21:52:44.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:43 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1445284371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T21:52:44.239 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:43 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1445284371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T21:52:44.240 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:44 smithi114 podman[47894]: 2024-09-13 21:52:44.129392108 +0000 UTC m=+1.555394270 container create 4371220e3ee6e72b1d177aca5c160df61ae7486f3e9f65cbbed515b5d264edaa (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:52:44.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:43 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1445284371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T21:52:44.929 INFO:teuthology.orchestra.run.smithi114.stdout:55834574896 2024-09-13T21:52:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:44 smithi121 ceph-mon[24681]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:45.306 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:44 smithi114 ceph-mon[23730]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:45.306 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:44 smithi114 ceph-mon[28528]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:45.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 podman[47894]: 2024-09-13 21:52:45.305359799 +0000 UTC m=+2.731361962 container init 4371220e3ee6e72b1d177aca5c160df61ae7486f3e9f65cbbed515b5d264edaa (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:52:45.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 podman[47894]: 2024-09-13 21:52:45.309226966 +0000 UTC m=+2.735229126 container start 4371220e3ee6e72b1d177aca5c160df61ae7486f3e9f65cbbed515b5d264edaa (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.319Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.319Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.321Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.114 port=9094 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.321Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.347Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.347Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.349Z caller=main.go:518 msg=Listening address=:9093 2024-09-13T21:52:45.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:45.349Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-13T21:52:45.875 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411371 for osd.2 2024-09-13T21:52:45.875 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:45 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3130017355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T21:52:46.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3130017355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T21:52:46.243 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:45 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3130017355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T21:52:46.441 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018911 got 128849018914 for osd.4 2024-09-13T21:52:46.441 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:46.830 INFO:teuthology.orchestra.run.smithi114.stdout:107374182438 2024-09-13T21:52:47.016 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574896 for osd.1 2024-09-13T21:52:47.016 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:47.110 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:46 smithi114 ceph-mon[28528]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:47.110 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:46 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2404488998' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T21:52:47.111 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:46 smithi114 ceph-mon[23730]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:47.111 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:46 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2404488998' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T21:52:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:46 smithi121 ceph-mon[24681]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:46 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2404488998' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T21:52:47.444 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:47.322Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000652104s 2024-09-13T21:52:48.028 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182434 got 107374182438 for osd.3 2024-09-13T21:52:48.028 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:48.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:48 smithi114 bash[47894]: 4371220e3ee6e72b1d177aca5c160df61ae7486f3e9f65cbbed515b5d264edaa 2024-09-13T21:52:48.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:48 smithi114 systemd[1]: Started Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:52:49.128 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[23730]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:49.128 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.128 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.128 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.128 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:52:49.128 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.129 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[28528]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:49.129 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.129 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.129 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.129 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:52:49.129 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.161 INFO:teuthology.orchestra.run.smithi114.stdout:163208757278 2024-09-13T21:52:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:48 smithi121 ceph-mon[24681]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:52:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:49.972 INFO:teuthology.orchestra.run.smithi114.stdout:231928234004 2024-09-13T21:52:50.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:49 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:52:50.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:49 smithi114 ceph-mon[23730]: Deploying daemon grafana.a on smithi121 2024-09-13T21:52:50.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:49 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2860681596' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T21:52:50.051 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:49 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:52:50.051 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:49 smithi114 ceph-mon[28528]: Deploying daemon grafana.a on smithi121 2024-09-13T21:52:50.051 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:49 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2860681596' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T21:52:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:49 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:52:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:49 smithi121 ceph-mon[24681]: Deploying daemon grafana.a on smithi121 2024-09-13T21:52:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:49 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2860681596' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T21:52:50.380 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757275 got 163208757278 for osd.5 2024-09-13T21:52:50.381 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:50.926 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:52:50] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:50 smithi121 ceph-mon[24681]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:50 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/199298251' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T21:52:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.217 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[28528]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:51.217 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/199298251' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T21:52:51.217 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.217 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.217 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.218 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[23730]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:51.218 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/199298251' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T21:52:51.218 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.218 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.218 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:51.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:52:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:51.773 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234000 got 231928234004 for osd.7 2024-09-13T21:52:51.773 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:51.773 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T21:52:51.773 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T21:52:52.192 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:52.319 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:52.319 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:52.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:53.142 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:53 smithi114 ceph-mon[23730]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:53.142 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:53 smithi114 ceph-mon[28528]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:53 smithi121 ceph-mon[24681]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:55.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:55 smithi114 ceph-mon[28528]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:55 smithi114 ceph-mon[23730]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:55 smithi121 ceph-mon[24681]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:55.406 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:52:55.407 INFO:teuthology.orchestra.run.smithi114.stderr:dumped all 2024-09-13T21:52:55.615 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:52:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:52:55.325Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003438519s 2024-09-13T21:52:55.918 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:52:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:56.234 INFO:teuthology.orchestra.run.smithi114.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-13T21:52:55.110238+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48892,"kb_used_data":6076,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912324,"statfs":{"total":767960285184,"available":767910219776,"internally_reserved":0,"allocated":6221824,"data_stored":3487591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001986"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T21:51:35.182892+0000","last_change":"2024-09-13T21:51:21.496034+0000","last_active":"2024-09-13T21:51:35.182892+0000","last_peered":"2024-09-13T21:51:35.182892+0000","last_clean":"2024-09-13T21:51:35.182892+0000","last_became_active":"2024-09-13T21:51:21.495642+0000","last_became_peered":"2024-09-13T21:51:21.495642+0000","last_unstale":"2024-09-13T21:51:35.182892+0000","last_undegraded":"2024-09-13T21:51:35.182892+0000","last_fullsized":"2024-09-13T21:51:35.182892+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T21:49:19.475731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T21:49:19.475731+0000","last_clean_scrub_stamp":"2024-09-13T21:49:19.475731+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.817105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.59999999999999998}]},{"osd":1,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.65000000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.68100000000000005}]},{"osd":3,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63800000000000001}]},{"osd":4,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.46400000000000002}]},{"osd":5,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45200000000000001}]},{"osd":6,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.441},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.47499999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495642,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":1110016,"data_stored":766446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.48699999999999999}]},{"osd":1,"last update":"Fri Sep 13 21:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.442}]},{"osd":2,"last update":"Fri Sep 13 21:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.45600000000000002}]},{"osd":3,"last update":"Fri Sep 13 21:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.53200000000000003}]},{"osd":4,"last update":"Fri Sep 13 21:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.34699999999999998}]},{"osd":5,"last update":"Fri Sep 13 21:51:53 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.504},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.47899999999999998}]},{"osd":7,"last update":"Fri Sep 13 21:52:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.51800000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6328,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:51:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.318},"min":{"1min":0.253,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.20000000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"min":{"1min":0.255,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.52400000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.214}]},{"osd":2,"last update":"Fri Sep 13 21:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.379,"15min":0.374},"min":{"1min":0.26700000000000002,"5min":0.19,"15min":0.19},"max":{"1min":0.51900000000000002,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54500000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.26400000000000001}]},{"osd":3,"last update":"Fri Sep 13 21:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.42799999999999999,"15min":0.43099999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.55100000000000005,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41399999999999998,"15min":0.41499999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.51100000000000001,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.27500000000000002}]},{"osd":4,"last update":"Fri Sep 13 21:52:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.376,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.74199999999999999,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78900000000000003,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.63400000000000001}]},{"osd":5,"last update":"Fri Sep 13 21:52:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.39400000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.68600000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71199999999999997,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.54100000000000004}]},{"osd":6,"last update":"Fri Sep 13 21:51:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.52900000000000003}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.57699999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 21:51:59 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.36399999999999999,"15min":0.35699999999999998},"min":{"1min":0.252,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.54900000000000004,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.29599999999999999,"5min":0.189,"15min":0.189},"max":{"1min":0.56499999999999995,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.312}]},{"osd":2,"last update":"Fri Sep 13 21:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39400000000000002,"15min":0.39500000000000002},"min":{"1min":0.27400000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.56000000000000005,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.25900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56100000000000005,"5min":0.629,"15min":0.629},"last":0.52800000000000002}]},{"osd":3,"last update":"Fri Sep 13 21:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.436,"15min":0.45100000000000001},"min":{"1min":0.29499999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.60399999999999998,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42999999999999999,"15min":0.442},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.58099999999999996,"5min":0.623,"15min":0.623},"last":0.36399999999999999}]},{"osd":4,"last update":"Fri Sep 13 21:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81799999999999995,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59099999999999997,"15min":0.59399999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.85099999999999998,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.35799999999999998}]},{"osd":5,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.44,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.623}]},{"osd":6,"last update":"Fri Sep 13 21:51:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54600000000000004}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.51700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6268,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738884,"statfs":{"total":95995035648,"available":95988617216,"internally_reserved":0,"allocated":577536,"data_stored":237576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38400000000000001,"15min":0.39500000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.55700000000000005,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.40200000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52400000000000002,"5min":0.622,"15min":0.622},"last":0.29199999999999998}]},{"osd":1,"last update":"Fri Sep 13 21:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.39600000000000002,"15min":0.40799999999999997},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.50800000000000001,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.246},{"interface":"front","average":{"1min":0.378,"5min":0.41599999999999998,"15min":0.42099999999999999},"min":{"1min":0.28799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.60599999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.30299999999999999}]},{"osd":3,"last update":"Fri Sep 13 21:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.40300000000000002,"15min":0.40699999999999997},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.57399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.313,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.58499999999999996,"5min":0.63,"15min":0.63},"last":0.33600000000000002}]},{"osd":4,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.55400000000000005,"15min":0.55100000000000005},"min":{"1min":0.44600000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73799999999999999,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.627},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.436,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.59999999999999998}]},{"osd":5,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56599999999999995,"15min":0.56499999999999995},"min":{"1min":0.41399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56399999999999995,"15min":0.56200000000000006},"min":{"1min":0.40899999999999997,"5min":0.371,"15min":0.371},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.60399999999999998}]},{"osd":6,"last update":"Fri Sep 13 21:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.53100000000000003}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57699999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.40600000000000003,"15min":0.42199999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.46400000000000002,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.31},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.41999999999999998,"15min":0.44},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.47999999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.26900000000000002}]},{"osd":1,"last update":"Fri Sep 13 21:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.40899999999999997,"15min":0.42199999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.442,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.436,"15min":0.45000000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.56799999999999995,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.25}]},{"osd":2,"last update":"Fri Sep 13 21:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.46800000000000003,"15min":0.48899999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.54100000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.45600000000000002,"15min":0.47199999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.42799999999999999,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.33200000000000002}]},{"osd":4,"last update":"Fri Sep 13 21:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.45000000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.48899999999999999}]},{"osd":5,"last update":"Fri Sep 13 21:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.60999999999999999,"15min":0.61699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82799999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.629,"15min":0.63400000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.40200000000000002}]},{"osd":6,"last update":"Fri Sep 13 21:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.38600000000000001}]},{"osd":7,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.376},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.43099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":0.66800000000000004,"15min":0.57299999999999995},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":14.356999999999999,"5min":14.356999999999999,"15min":14.356999999999999},"last":0.499},{"interface":"front","average":{"1min":1.2989999999999999,"5min":0.72299999999999998,"15min":0.627},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":14.347,"5min":14.347,"15min":14.347},"last":0.42499999999999999}]},{"osd":1,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.52400000000000002,"15min":0.52900000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.63300000000000001,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.442},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66700000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.38900000000000001}]},{"osd":2,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.75700000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.51800000000000002}]},{"osd":3,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.78200000000000003,"15min":0.82699999999999996},"min":{"1min":0.373,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.78400000000000003,"5min":5.6989999999999998,"15min":5.6989999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70099999999999996,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.56799999999999995}]},{"osd":5,"last update":"Fri Sep 13 21:52:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.42299999999999999,"15min":0.41699999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.42999999999999999,"15min":0.42399999999999999},"min":{"1min":0.33800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.435}]},{"osd":6,"last update":"Fri Sep 13 21:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.46700000000000003}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.58899999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5684,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739468,"statfs":{"total":95995035648,"available":95989215232,"internally_reserved":0,"allocated":569344,"data_stored":237038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.60699999999999998,"15min":0.53500000000000003},"min":{"1min":0.32100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.826000000000001,"5min":13.826000000000001,"15min":13.826000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.036,"5min":0.60299999999999998,"15min":0.53100000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.839,"5min":13.839,"15min":13.839},"last":0.49099999999999999}]},{"osd":1,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.60199999999999998,"15min":0.53400000000000003},"min":{"1min":0.33600000000000002,"5min":0.313,"15min":0.313},"max":{"1min":13.853999999999999,"5min":13.853999999999999,"15min":13.853999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.05,"5min":0.61099999999999999,"15min":0.53800000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.803000000000001,"5min":13.803000000000001,"15min":13.803000000000001},"last":0.57499999999999996}]},{"osd":2,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.439},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.48899999999999999,"15min":0.48599999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.59999999999999998}]},{"osd":3,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.55900000000000005}]},{"osd":4,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.314},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.40100000000000002,"15min":0.38800000000000001},"min":{"1min":0.34200000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.35799999999999998}]},{"osd":6,"last update":"Fri Sep 13 21:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.33200000000000002}]},{"osd":7,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.373},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.379}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T21:52:56.236 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T21:52:56.460 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:52:56.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:56 smithi114 ceph-mon[23730]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:56.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:56 smithi114 ceph-mon[23730]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:52:56.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:56.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:56 smithi114 ceph-mon[28528]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:56.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:56 smithi114 ceph-mon[28528]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:52:56.847 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:56 smithi121 ceph-mon[24681]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:57.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:56 smithi121 ceph-mon[24681]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:52:57.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:58.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:57 smithi121 systemd[1]: Starting Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:52:58.443 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 podman[43277]: 2024-09-13 21:52:58.073889271 +0000 UTC m=+0.033610788 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-09-13T21:52:58.443 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 podman[43277]: 2024-09-13 21:52:58.191851482 +0000 UTC m=+0.151572984 container create b8bdbef3d4898f0ef5ad05a3d92b443991e80634f7cb55600a9ce1e4d87b6ac5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, maintainer=Paul Cuzner , architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, vcs-type=git) 2024-09-13T21:52:58.443 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 podman[43277]: 2024-09-13 21:52:58.410765353 +0000 UTC m=+0.370486864 container init b8bdbef3d4898f0ef5ad05a3d92b443991e80634f7cb55600a9ce1e4d87b6ac5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, io.openshift.expose-services=, description=Ceph Grafana Container, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 podman[43277]: 2024-09-13 21:52:58.413610191 +0000 UTC m=+0.373331701 container start b8bdbef3d4898f0ef5ad05a3d92b443991e80634f7cb55600a9ce1e4d87b6ac5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, description=Ceph Grafana Container, io.buildah.version=1.24.2, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vcs-type=git, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-13T21:52:58.444 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-13T21:52:58.445 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-13T21:52:58.445 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-13T21:52:58.445 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="App mode production" logger=settings 2024-09-13T21:52:58.445 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-13T21:52:58.445 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T21:52:58.445 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-09-13T21:52:58.445 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-13T21:52:58.694 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-09-13T21:52:58.695 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-09-13T21:52:58.696 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T21:52:58.697 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-09-13T21:52:58.697 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-09-13T21:52:58.697 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-09-13T21:52:58.698 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-09-13T21:52:58.699 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-09-13T21:52:58.700 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-09-13T21:52:58.701 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-13T21:52:58.702 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 bash[43277]: b8bdbef3d4898f0ef5ad05a3d92b443991e80634f7cb55600a9ce1e4d87b6ac5 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 systemd[1]: Started Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-09-13T21:52:58.703 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-09-13T21:52:58.704 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-09-13T21:52:58.705 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-09-13T21:52:58.706 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-09-13T21:52:58.707 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-09-13T21:52:58.708 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-09-13T21:52:58.709 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-09-13T21:52:58.710 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-09-13T21:52:58.788 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[23730]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:58.788 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:58.788 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[28528]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:58.789 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:59.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-mon[24681]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:52:59.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:59.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:59.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:59.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:59.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-09-13T21:52:59.004 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-09-13T21:52:59.005 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T21:52:59.006 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-09-13T21:52:59.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-09-13T21:52:59.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-09-13T21:52:59.009 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-09-13T21:52:59.010 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T21:52:59.010 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-09-13T21:52:59.010 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-09-13T21:52:59.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-09-13T21:52:59.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-09-13T21:52:59.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-09-13T21:52:59.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T21:52:59.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-09-13T21:52:59.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-09-13T21:52:59.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-09-13T21:52:59.012 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-09-13T21:52:59.013 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T21:52:59.014 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-09-13T21:52:59.015 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-09-13T21:52:59.016 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-09-13T21:52:59.017 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-09-13T21:52:59.018 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-09-13T21:52:59.019 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-09-13T21:52:59.020 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=447.063965ms 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:58+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:59+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:59+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:59+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-13T21:52:59.021 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:59+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T21:52:59.270 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:52:59.272 INFO:teuthology.orchestra.run.smithi114.stderr:dumped all 2024-09-13T21:52:59.287 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T21:52:59+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-13T21:52:59.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:52:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:52:59.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[23730]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[23730]: Reconfiguring daemon alertmanager.a on smithi114 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[28528]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[28528]: Reconfiguring daemon alertmanager.a on smithi114 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:52:59.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:52:59 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-mon[24681]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T21:53:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-mon[24681]: Reconfiguring daemon alertmanager.a on smithi114 2024-09-13T21:53:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:52:59 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:00 smithi121 ceph-mon[24681]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:00 smithi121 ceph-mon[24681]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:53:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:00 smithi114 ceph-mon[23730]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:00 smithi114 ceph-mon[23730]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:53:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:01.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:00 smithi114 ceph-mon[28528]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:00 smithi114 ceph-mon[28528]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:53:01.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:01.294 INFO:teuthology.orchestra.run.smithi114.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-09-13T21:52:59.110960+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48892,"kb_used_data":6076,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912324,"statfs":{"total":767960285184,"available":767910219776,"internally_reserved":0,"allocated":6221824,"data_stored":3487591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002042"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T21:51:35.182892+0000","last_change":"2024-09-13T21:51:21.496034+0000","last_active":"2024-09-13T21:51:35.182892+0000","last_peered":"2024-09-13T21:51:35.182892+0000","last_clean":"2024-09-13T21:51:35.182892+0000","last_became_active":"2024-09-13T21:51:21.495642+0000","last_became_peered":"2024-09-13T21:51:21.495642+0000","last_unstale":"2024-09-13T21:51:35.182892+0000","last_undegraded":"2024-09-13T21:51:35.182892+0000","last_fullsized":"2024-09-13T21:51:35.182892+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T21:49:19.475731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T21:49:19.475731+0000","last_clean_scrub_stamp":"2024-09-13T21:49:19.475731+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:18.817105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234006,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.47299999999999998}]},{"osd":1,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.49399999999999999}]},{"osd":2,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52900000000000003}]},{"osd":3,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.504},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.48999999999999999}]},{"osd":4,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.38500000000000001}]},{"osd":5,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.442}]},{"osd":6,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.46100000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495643,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":1110016,"data_stored":766446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70099999999999996,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.54000000000000004,"15min":0.54600000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.748,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.53800000000000003}]},{"osd":1,"last update":"Fri Sep 13 21:52:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.55100000000000005,"15min":0.55400000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.75800000000000001,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.54800000000000004,"15min":0.55100000000000005},"min":{"1min":0.38700000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.76000000000000001,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.60299999999999998}]},{"osd":2,"last update":"Fri Sep 13 21:52:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.38200000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.81100000000000005,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.55700000000000005,"15min":0.56399999999999995},"min":{"1min":0.36699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77300000000000002,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.58099999999999996}]},{"osd":3,"last update":"Fri Sep 13 21:52:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.57499999999999996,"15min":0.57999999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79800000000000004,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.55800000000000005},"min":{"1min":0.443,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78000000000000003,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.502}]},{"osd":4,"last update":"Fri Sep 13 21:52:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.47099999999999997,"15min":0.46400000000000002},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65700000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.34699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.55600000000000005}]},{"osd":5,"last update":"Fri Sep 13 21:52:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.504,"15min":0.499},"min":{"1min":0.38,"5min":0.307,"15min":0.307},"max":{"1min":0.69599999999999995,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.36399999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.70799999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.70399999999999996}]},{"osd":7,"last update":"Fri Sep 13 21:52:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.627},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.73399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6328,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:51:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.318},"min":{"1min":0.253,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"min":{"1min":0.255,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.52400000000000002,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.26800000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.379,"15min":0.374},"min":{"1min":0.26700000000000002,"5min":0.19,"15min":0.19},"max":{"1min":0.51900000000000002,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54500000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.30199999999999999}]},{"osd":3,"last update":"Fri Sep 13 21:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.42799999999999999,"15min":0.43099999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.55100000000000005,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41399999999999998,"15min":0.41499999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.51100000000000001,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.43099999999999999}]},{"osd":4,"last update":"Fri Sep 13 21:52:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.376,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.74199999999999999,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78900000000000003,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.41599999999999998}]},{"osd":5,"last update":"Fri Sep 13 21:52:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.39400000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.68600000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71199999999999997,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.44}]},{"osd":6,"last update":"Fri Sep 13 21:51:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.60499999999999998}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.501}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 21:51:59 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.36399999999999999,"15min":0.35699999999999998},"min":{"1min":0.252,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.54900000000000004,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.29599999999999999,"5min":0.189,"15min":0.189},"max":{"1min":0.56499999999999995,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.37}]},{"osd":2,"last update":"Fri Sep 13 21:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39400000000000002,"15min":0.39500000000000002},"min":{"1min":0.27400000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.56000000000000005,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.311},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.25900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56100000000000005,"5min":0.629,"15min":0.629},"last":0.41599999999999998}]},{"osd":3,"last update":"Fri Sep 13 21:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.436,"15min":0.45100000000000001},"min":{"1min":0.29499999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.60399999999999998,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42999999999999999,"15min":0.442},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.58099999999999996,"5min":0.623,"15min":0.623},"last":0.39300000000000002}]},{"osd":4,"last update":"Fri Sep 13 21:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81799999999999995,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.44},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59099999999999997,"15min":0.59399999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.85099999999999998,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.48399999999999999}]},{"osd":5,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.44,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.66600000000000004}]},{"osd":6,"last update":"Fri Sep 13 21:51:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.47099999999999997}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.49299999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6268,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738884,"statfs":{"total":95995035648,"available":95988617216,"internally_reserved":0,"allocated":577536,"data_stored":237576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38400000000000001,"15min":0.39500000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.55700000000000005,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.40200000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52400000000000002,"5min":0.622,"15min":0.622},"last":0.28699999999999998}]},{"osd":1,"last update":"Fri Sep 13 21:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.39600000000000002,"15min":0.40799999999999997},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.50800000000000001,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.41599999999999998,"15min":0.42099999999999999},"min":{"1min":0.28799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.60599999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.33000000000000002}]},{"osd":3,"last update":"Fri Sep 13 21:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.40300000000000002,"15min":0.40699999999999997},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.57399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.313,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.58499999999999996,"5min":0.63,"15min":0.63},"last":0.375}]},{"osd":4,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.55400000000000005,"15min":0.55100000000000005},"min":{"1min":0.44600000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73799999999999999,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.436,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.54000000000000004}]},{"osd":5,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56599999999999995,"15min":0.56499999999999995},"min":{"1min":0.41399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56399999999999995,"15min":0.56200000000000006},"min":{"1min":0.40899999999999997,"5min":0.371,"15min":0.371},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.42999999999999999}]},{"osd":6,"last update":"Fri Sep 13 21:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.45100000000000001}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.50800000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182441,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.40600000000000003,"15min":0.42199999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.46400000000000002,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.41999999999999998,"15min":0.44},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.47999999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.441}]},{"osd":1,"last update":"Fri Sep 13 21:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.40899999999999997,"15min":0.42199999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.442,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.436,"15min":0.45000000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.56799999999999995,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.42599999999999999}]},{"osd":2,"last update":"Fri Sep 13 21:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.46800000000000003,"15min":0.48899999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.54100000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.45600000000000002,"15min":0.47199999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.42799999999999999,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.40600000000000003}]},{"osd":4,"last update":"Fri Sep 13 21:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.45000000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.58199999999999996}]},{"osd":5,"last update":"Fri Sep 13 21:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.60999999999999999,"15min":0.61699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82799999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.629,"15min":0.63400000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.625}]},{"osd":6,"last update":"Fri Sep 13 21:51:55 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.58999999999999997}]},{"osd":7,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.64700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018917,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":0.66800000000000004,"15min":0.57299999999999995},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":14.356999999999999,"5min":14.356999999999999,"15min":14.356999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.2989999999999999,"5min":0.72299999999999998,"15min":0.627},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":14.347,"5min":14.347,"15min":14.347},"last":0.48699999999999999}]},{"osd":1,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.52400000000000002,"15min":0.52900000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.63300000000000001,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66700000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.47399999999999998}]},{"osd":2,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.75700000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.44800000000000001}]},{"osd":3,"last update":"Fri Sep 13 21:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.78200000000000003,"15min":0.82699999999999996},"min":{"1min":0.373,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.78400000000000003,"5min":5.6989999999999998,"15min":5.6989999999999998},"last":0.374},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70099999999999996,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.50600000000000001}]},{"osd":5,"last update":"Fri Sep 13 21:52:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.42299999999999999,"15min":0.41699999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.313},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.42999999999999999,"15min":0.42399999999999999},"min":{"1min":0.33800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.34799999999999998}]},{"osd":6,"last update":"Fri Sep 13 21:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.433}]},{"osd":7,"last update":"Fri Sep 13 21:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.499},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.52700000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757280,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5684,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739468,"statfs":{"total":95995035648,"available":95989215232,"internally_reserved":0,"allocated":569344,"data_stored":237038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.60699999999999998,"15min":0.53500000000000003},"min":{"1min":0.32100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.826000000000001,"5min":13.826000000000001,"15min":13.826000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.036,"5min":0.60299999999999998,"15min":0.53100000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.839,"5min":13.839,"15min":13.839},"last":0.498}]},{"osd":1,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.60199999999999998,"15min":0.53400000000000003},"min":{"1min":0.33600000000000002,"5min":0.313,"15min":0.313},"max":{"1min":13.853999999999999,"5min":13.853999999999999,"15min":13.853999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.05,"5min":0.61099999999999999,"15min":0.53800000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.803000000000001,"5min":13.803000000000001,"15min":13.803000000000001},"last":0.51800000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.48899999999999999,"15min":0.48599999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.48299999999999998}]},{"osd":3,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.56499999999999995}]},{"osd":4,"last update":"Fri Sep 13 21:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.372},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.40100000000000002,"15min":0.38800000000000001},"min":{"1min":0.34200000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.432}]},{"osd":6,"last update":"Fri Sep 13 21:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.505}]},{"osd":7,"last update":"Fri Sep 13 21:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.57299999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T21:53:01.296 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T21:53:01.296 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T21:53:01.296 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T21:53:01.296 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T21:53:02.947 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:02 smithi121 ceph-mon[24681]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:03.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:02 smithi114 ceph-mon[23730]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:03.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:02 smithi114 ceph-mon[28528]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:03.976 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:03 smithi114 systemd[1]: Stopping Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:53:03.976 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:03 smithi114 bash[49412]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager.a" found: no such container 2024-09-13T21:53:03.976 INFO:teuthology.orchestra.run.smithi114.stderr:Inferring config /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T21:53:04.276 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[48028]: level=info ts=2024-09-13T21:53:03.975Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-13T21:53:04.276 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:03 smithi114 podman[49423]: 2024-09-13 21:53:03.995493906 +0000 UTC m=+0.140435198 container died 4371220e3ee6e72b1d177aca5c160df61ae7486f3e9f65cbbed515b5d264edaa (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:04.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:04 smithi114 podman[49423]: 2024-09-13 21:53:04.453416117 +0000 UTC m=+0.598357402 container cleanup 4371220e3ee6e72b1d177aca5c160df61ae7486f3e9f65cbbed515b5d264edaa (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:04.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:04 smithi114 bash[49423]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T21:53:04.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:04 smithi114 bash[49461]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager.a" found: no such container 2024-09-13T21:53:04.774 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:04 smithi121 ceph-mon[24681]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:04.920 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:04 smithi114 ceph-mon[23730]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:04.921 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:04 smithi114 ceph-mon[28528]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:04.921 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:53:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:05.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:05 smithi114 podman[49448]: 2024-09-13 21:53:05.557529993 +0000 UTC m=+1.560762045 container remove 4371220e3ee6e72b1d177aca5c160df61ae7486f3e9f65cbbed515b5d264edaa (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:05.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:05 smithi114 podman[49448]: 2024-09-13 21:53:05.658226363 +0000 UTC m=+1.661458422 volume remove 1c6a1d879011173dbe999ef91b1e5438e91c2bd02206880285b455bcdbeedd2c 2024-09-13T21:53:05.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:05 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T21:53:05.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:05 smithi114 systemd[1]: Stopped Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:53:05.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:05 smithi114 systemd[1]: Starting Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:53:06.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:06.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:06.500 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 podman[49566]: 2024-09-13 21:53:06.081657245 +0000 UTC m=+0.196585902 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-09-13T21:53:06.500 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 podman[49566]: 2024-09-13 21:53:06.20406747 +0000 UTC m=+0.318996162 volume create 292785bab4af9a9116296afbfe623664c9d413d2803f9000b988f4ba97741139 2024-09-13T21:53:06.500 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 podman[49566]: 2024-09-13 21:53:06.304374639 +0000 UTC m=+0.419303287 container create 9bf04fad0c856206e4a2aa608704ba029fcb1914e48a55eb7cdff65b62544b05 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 podman[49566]: 2024-09-13 21:53:06.598676537 +0000 UTC m=+0.713605203 container init 9bf04fad0c856206e4a2aa608704ba029fcb1914e48a55eb7cdff65b62544b05 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 podman[49566]: 2024-09-13 21:53:06.60149575 +0000 UTC m=+0.716424398 container start 9bf04fad0c856206e4a2aa608704ba029fcb1914e48a55eb7cdff65b62544b05 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.611Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.612Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.613Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.114 port=9094 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.614Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.639Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T21:53:06.868 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.639Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T21:53:06.869 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.641Z caller=main.go:518 msg=Listening address=:9093 2024-09-13T21:53:06.869 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:06.641Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-13T21:53:06.869 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 bash[49566]: 9bf04fad0c856206e4a2aa608704ba029fcb1914e48a55eb7cdff65b62544b05 2024-09-13T21:53:06.869 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:06 smithi114 systemd[1]: Started Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:53:06.984 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:06 smithi121 ceph-mon[24681]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:06.984 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:06 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:07.176 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-mon[23730]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:07.176 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:07.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-mon[28528]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:07.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:06 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:07.189 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:53:07.189 INFO:teuthology.orchestra.run.smithi114.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T21:53:07.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:53:07] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:07.813 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T21:53:07.813 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T21:53:07.814 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T21:53:07.827 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi114.front.sepia.ceph.com 2024-09-13T21:53:07.827 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-09-13T21:53:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[23730]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[23730]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1245308943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[28528]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[28528]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1245308943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:07 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:07 smithi121 ceph-mon[24681]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T21:53:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:07 smithi121 ceph-mon[24681]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T21:53:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:07 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1245308943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T21:53:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:07 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:07 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:08.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:08.615Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000994507s 2024-09-13T21:53:09.196 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:08 smithi121 ceph-mon[24681]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:09.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:08 smithi114 ceph-mon[23730]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:08 smithi114 ceph-mon[28528]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 systemd[1]: Stopping Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 bash[44086]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus.a" found: no such container 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.211Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.211Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-09-13T21:53:10.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T21:53:10.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T21:53:10.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-09-13T21:53:10.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[40951]: ts=2024-09-13T21:53:10.212Z caller=main.go:1066 level=info msg="See you next time!" 2024-09-13T21:53:10.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 podman[44097]: 2024-09-13 21:53:10.221980596 +0000 UTC m=+0.088329227 container died 05dcec8c97fe3e72edd1e61c9f15283d98676233aef36e14e018e512bda677cb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:10.838 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 podman[44097]: 2024-09-13 21:53:10.636531786 +0000 UTC m=+0.502880414 container cleanup 05dcec8c97fe3e72edd1e61c9f15283d98676233aef36e14e018e512bda677cb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:10.838 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 bash[44097]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T21:53:10.838 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 bash[44123]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus.a" found: no such container 2024-09-13T21:53:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:10 smithi114 ceph-mon[23730]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:10 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1951109152' entity='client.admin' 2024-09-13T21:53:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:10 smithi114 ceph-mon[28528]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:10 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1951109152' entity='client.admin' 2024-09-13T21:53:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-mon[24681]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:10 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1951109152' entity='client.admin' 2024-09-13T21:53:11.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 podman[44110]: 2024-09-13 21:53:10.837769839 +0000 UTC m=+0.612730028 container remove 05dcec8c97fe3e72edd1e61c9f15283d98676233aef36e14e018e512bda677cb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:11.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T21:53:11.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 systemd[1]: Stopped Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:53:11.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:10 smithi121 systemd[1]: Starting Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:53:11.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 podman[44184]: 2024-09-13 21:53:11.047200819 +0000 UTC m=+0.094249774 container create baadd3f398d493c26694b92523ce6cdf29b7b85d74bd7484cd47434586a967b9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:11.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 podman[44184]: 2024-09-13 21:53:10.969454636 +0000 UTC m=+0.016503599 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-09-13T21:53:11.179 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T21:53:11.190 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi114.front.sepia.ceph.com 2024-09-13T21:53:11.190 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-09-13T21:53:11.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 podman[44184]: 2024-09-13 21:53:11.333279875 +0000 UTC m=+0.380328836 container init baadd3f398d493c26694b92523ce6cdf29b7b85d74bd7484cd47434586a967b9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:11.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 podman[44184]: 2024-09-13 21:53:11.336150727 +0000 UTC m=+0.383199679 container start baadd3f398d493c26694b92523ce6cdf29b7b85d74bd7484cd47434586a967b9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T21:53:11.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.359Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-13T21:53:11.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.359Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-09-13T21:53:11.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.359Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-09-13T21:53:11.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.359Z caller=main.go:518 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi121 (none))" 2024-09-13T21:53:11.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.359Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T21:53:11.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.359Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T21:53:11.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.360Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T21:53:11.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.361Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-09-13T21:53:11.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.361Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-09-13T21:53:11.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.364Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T21:53:11.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.364Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.678µs 2024-09-13T21:53:11.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.364Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T21:53:11.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.366Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T21:53:11.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.366Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T21:53:11.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.366Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.987µs wal_replay_duration=1.889437ms total_replay_duration=1.929436ms 2024-09-13T21:53:11.624 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.367Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T21:53:11.625 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.367Z caller=main.go:947 level=info msg="TSDB started" 2024-09-13T21:53:11.625 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.367Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T21:53:11.625 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.383Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.425414ms db_storage=593ns remote_storage=1.424µs web_handler=295ns query_engine=549ns scrape=352.501µs scrape_sd=21.939µs notify=17.961µs notify_sd=5.789µs rules=14.810569ms 2024-09-13T21:53:11.625 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T21:53:11.383Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-09-13T21:53:11.625 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 bash[44184]: baadd3f398d493c26694b92523ce6cdf29b7b85d74bd7484cd47434586a967b9 2024-09-13T21:53:11.625 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 21:53:11 smithi121 systemd[1]: Started Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:53:12.539 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: pgmap v51: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:12.539 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.539 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:53:12.539 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:53:12.539 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.114:9093"}]: dispatch 2024-09-13T21:53:12.539 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.114:9093"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.121:3000"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.121:3000"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:12.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:12.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: pgmap v51: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:12.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:53:12.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:53:12.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.114:9093"}]: dispatch 2024-09-13T21:53:12.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.114:9093"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.121:3000"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.121:3000"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:12.545 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: pgmap v51: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:12.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.114:9093"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.114:9093"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.121:3000"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.121:3000"}]: dispatch 2024-09-13T21:53:12.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:12.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:12.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:12.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:12.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:12.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:14.773 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:14 smithi114 ceph-mon[23730]: pgmap v52: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:14.773 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:14 smithi114 ceph-mon[28528]: pgmap v52: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:14.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:14 smithi121 ceph-mon[24681]: pgmap v52: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:16.619 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:16 smithi114 ceph-mon[23730]: pgmap v53: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:16.619 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:16 smithi114 ceph-mon[28528]: pgmap v53: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:16.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:16 smithi121 ceph-mon[24681]: pgmap v53: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:17.074 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:53:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T21:53:16.618Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004045067s 2024-09-13T21:53:17.551 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T21:53:17.551 INFO:teuthology.orchestra.run.smithi114.stdout: "id": "88bef8de-f1e8-4769-bf53-735644f6765d", 2024-09-13T21:53:17.552 INFO:teuthology.orchestra.run.smithi114.stdout: "name": "r", 2024-09-13T21:53:17.552 INFO:teuthology.orchestra.run.smithi114.stdout: "current_period": "7a9c15ed-8fc6-472c-b369-043b1f02786e", 2024-09-13T21:53:17.552 INFO:teuthology.orchestra.run.smithi114.stdout: "epoch": 1 2024-09-13T21:53:17.552 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T21:53:17.775 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:17 smithi114 ceph-mon[23730]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T21:53:17.776 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:17 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/349545807' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T21:53:17.776 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:17 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T21:53:17.776 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:17 smithi114 ceph-mon[28528]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T21:53:17.776 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:17 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/349545807' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T21:53:17.777 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:17 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T21:53:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:17 smithi121 ceph-mon[24681]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T21:53:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:17 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/349545807' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T21:53:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:17 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-13T21:53:18.059 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:53:17] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:18.410 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-09-13T21:53:18.519 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:53:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:53:18.713 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[23730]: pgmap v55: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:18.713 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-13T21:53:18.714 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[23730]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T21:53:18.714 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:18.714 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:18.714 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[28528]: pgmap v55: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:18.714 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-13T21:53:18.715 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[28528]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T21:53:18.715 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:18.715 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:18.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:18 smithi121 ceph-mon[24681]: pgmap v55: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:18.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:18 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-13T21:53:18.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:18 smithi121 ceph-mon[24681]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T21:53:18.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:18.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:19.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:19 smithi114 ceph-mon[23730]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T21:53:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:19 smithi114 ceph-mon[28528]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T21:53:19.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:19 smithi121 ceph-mon[24681]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T21:53:20.143 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "id": "c40f7123-726f-495d-a85a-307614441c03", 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "name": "default", 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "api_name": "default", 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "is_master": "true", 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "endpoints": [], 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "hostnames": [], 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "hostnames_s3website": [], 2024-09-13T21:53:20.144 INFO:teuthology.orchestra.run.smithi114.stdout: "master_zone": "", 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout: "zones": [], 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout: "placement_targets": [], 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout: "default_placement": "", 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout: "realm_id": "88bef8de-f1e8-4769-bf53-735644f6765d", 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout: "sync_policy": { 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout: "groups": [] 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:20.145 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T21:53:20.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:20 smithi114 ceph-mon[23730]: pgmap v58: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:20.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:20 smithi114 ceph-mon[28528]: pgmap v58: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:20.852 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-09-13T21:53:20.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:20 smithi121 ceph-mon[24681]: pgmap v58: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:53:22.622 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T21:53:22.622 INFO:teuthology.orchestra.run.smithi114.stdout: "id": "e5aef97f-ca92-4aa2-815e-9e832eff76cf", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "name": "z", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "domain_root": "z.rgw.meta:root", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "control_pool": "z.rgw.control", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "gc_pool": "z.rgw.log:gc", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "lc_pool": "z.rgw.log:lc", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "log_pool": "z.rgw.log", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "roles_pool": "z.rgw.meta:roles", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-09-13T21:53:22.623 INFO:teuthology.orchestra.run.smithi114.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "otp_pool": "z.rgw.otp", 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "system_key": { 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "access_key": "", 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "secret_key": "" 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "placement_pools": [ 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: { 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "key": "default-placement", 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "val": { 2024-09-13T21:53:22.624 INFO:teuthology.orchestra.run.smithi114.stdout: "index_pool": "z.rgw.buckets.index", 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: "storage_classes": { 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: "STANDARD": { 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: "data_pool": "z.rgw.buckets.data" 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: "index_type": 0 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: ], 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: "realm_id": "88bef8de-f1e8-4769-bf53-735644f6765d", 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout: "notif_pool": "z.rgw.log:notif" 2024-09-13T21:53:22.625 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T21:53:22.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:22 smithi114 ceph-mon[28528]: pgmap v59: 33 pgs: 25 active+clean, 8 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 1023 B/s wr, 1 op/s 2024-09-13T21:53:22.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:22 smithi114 ceph-mon[23730]: pgmap v59: 33 pgs: 25 active+clean, 8 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 1023 B/s wr, 1 op/s 2024-09-13T21:53:22.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:22 smithi121 ceph-mon[24681]: pgmap v59: 33 pgs: 25 active+clean, 8 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 1023 B/s wr, 1 op/s 2024-09-13T21:53:23.285 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-09-13T21:53:24.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:24 smithi121 ceph-mon[24681]: pgmap v60: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-09-13T21:53:24.723 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:24 smithi114 ceph-mon[23730]: pgmap v60: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-09-13T21:53:24.724 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:24 smithi114 ceph-mon[28528]: pgmap v60: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-09-13T21:53:25.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:25] "GET /metrics HTTP/1.1" 200 192565 "" "Prometheus/2.33.4" 2024-09-13T21:53:26.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:26 smithi114 ceph-mon[23730]: pgmap v61: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T21:53:26.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:26 smithi114 ceph-mon[23730]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T21:53:26.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:26 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T21:53:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:26 smithi114 ceph-mon[28528]: pgmap v61: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T21:53:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:26 smithi114 ceph-mon[28528]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T21:53:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:26 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T21:53:26.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:26 smithi121 ceph-mon[24681]: pgmap v61: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T21:53:26.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:26 smithi121 ceph-mon[24681]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T21:53:26.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:26 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-13T21:53:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:27 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-13T21:53:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:27 smithi121 ceph-mon[24681]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T21:53:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:27 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:27 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:27.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:53:27] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[28528]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[23730]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:27 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:28.579 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:53:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:53:28.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:28 smithi121 ceph-mon[24681]: pgmap v64: 65 pgs: 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-13T21:53:28.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:28 smithi121 ceph-mon[24681]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T21:53:28.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:28 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T21:53:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:28 smithi114 ceph-mon[28528]: pgmap v64: 65 pgs: 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-13T21:53:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:28 smithi114 ceph-mon[28528]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T21:53:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:28 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T21:53:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:28 smithi114 ceph-mon[23730]: pgmap v64: 65 pgs: 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-13T21:53:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:28 smithi114 ceph-mon[23730]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T21:53:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:28 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-13T21:53:29.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:29 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-13T21:53:29.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:29 smithi121 ceph-mon[24681]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T21:53:29.903 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:29 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-13T21:53:29.903 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:29 smithi114 ceph-mon[23730]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T21:53:29.903 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:29 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/866341140' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-13T21:53:29.903 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:29 smithi114 ceph-mon[28528]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T21:53:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:53:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:30.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[23730]: pgmap v67: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T21:53:30.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[23730]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[28528]: pgmap v67: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[28528]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:30 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:30 smithi121 ceph-mon[24681]: pgmap v67: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T21:53:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:30 smithi121 ceph-mon[24681]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T21:53:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:30 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-13T21:53:30.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:30 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:30.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:30 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:31 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-13T21:53:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:31 smithi121 ceph-mon[24681]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T21:53:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:31 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-13T21:53:31.983 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:31 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-13T21:53:31.983 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:31 smithi114 ceph-mon[23730]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T21:53:31.983 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:31 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-13T21:53:31.983 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:31 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-13T21:53:31.983 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:31 smithi114 ceph-mon[28528]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T21:53:31.983 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:31 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-13T21:53:32.294 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:53:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:32.665 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T21:53:32.665 INFO:teuthology.orchestra.run.smithi114.stdout: "id": "8656ed70-f9c4-462d-864a-63ff01308fa4", 2024-09-13T21:53:32.665 INFO:teuthology.orchestra.run.smithi114.stdout: "epoch": 1, 2024-09-13T21:53:32.665 INFO:teuthology.orchestra.run.smithi114.stdout: "predecessor_uuid": "7a9c15ed-8fc6-472c-b369-043b1f02786e", 2024-09-13T21:53:32.665 INFO:teuthology.orchestra.run.smithi114.stdout: "sync_status": [], 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "period_map": { 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "id": "8656ed70-f9c4-462d-864a-63ff01308fa4", 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "zonegroups": [ 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: { 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "id": "c40f7123-726f-495d-a85a-307614441c03", 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "name": "default", 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "api_name": "default", 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "is_master": "true", 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "endpoints": [], 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "hostnames": [], 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "hostnames_s3website": [], 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "master_zone": "e5aef97f-ca92-4aa2-815e-9e832eff76cf", 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "zones": [ 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: { 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "id": "e5aef97f-ca92-4aa2-815e-9e832eff76cf", 2024-09-13T21:53:32.666 INFO:teuthology.orchestra.run.smithi114.stdout: "name": "z", 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "endpoints": [], 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "log_meta": "false", 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "log_data": "false", 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "bucket_index_max_shards": 11, 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "read_only": "false", 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "tier_type": "", 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "sync_from_all": "true", 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "sync_from": [], 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "redirect_zone": "" 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: ], 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: "placement_targets": [ 2024-09-13T21:53:32.667 INFO:teuthology.orchestra.run.smithi114.stdout: { 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "name": "default-placement", 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "tags": [], 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "storage_classes": [ 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "STANDARD" 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: ] 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: ], 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "default_placement": "default-placement", 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "realm_id": "88bef8de-f1e8-4769-bf53-735644f6765d", 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "sync_policy": { 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "groups": [] 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: ], 2024-09-13T21:53:32.668 INFO:teuthology.orchestra.run.smithi114.stdout: "short_zone_ids": [ 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: { 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "key": "e5aef97f-ca92-4aa2-815e-9e832eff76cf", 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "val": 579931791 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: ] 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "master_zonegroup": "c40f7123-726f-495d-a85a-307614441c03", 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "master_zone": "e5aef97f-ca92-4aa2-815e-9e832eff76cf", 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "period_config": { 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "bucket_quota": { 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "enabled": false, 2024-09-13T21:53:32.669 INFO:teuthology.orchestra.run.smithi114.stdout: "check_on_raw": false, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_size": -1, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_size_kb": 0, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_objects": -1 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "user_quota": { 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "enabled": false, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "check_on_raw": false, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_size": -1, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_size_kb": 0, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_objects": -1 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "user_ratelimit": { 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_read_ops": 0, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_write_ops": 0, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_read_bytes": 0, 2024-09-13T21:53:32.670 INFO:teuthology.orchestra.run.smithi114.stdout: "max_write_bytes": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "enabled": false 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "bucket_ratelimit": { 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_read_ops": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_write_ops": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_read_bytes": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_write_bytes": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "enabled": false 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "anonymous_ratelimit": { 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_read_ops": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_write_ops": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_read_bytes": 0, 2024-09-13T21:53:32.671 INFO:teuthology.orchestra.run.smithi114.stdout: "max_write_bytes": 0, 2024-09-13T21:53:32.672 INFO:teuthology.orchestra.run.smithi114.stdout: "enabled": false 2024-09-13T21:53:32.672 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:53:32.672 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:53:32.672 INFO:teuthology.orchestra.run.smithi114.stdout: "realm_id": "88bef8de-f1e8-4769-bf53-735644f6765d", 2024-09-13T21:53:32.672 INFO:teuthology.orchestra.run.smithi114.stdout: "realm_name": "r", 2024-09-13T21:53:32.672 INFO:teuthology.orchestra.run.smithi114.stdout: "realm_epoch": 2 2024-09-13T21:53:32.672 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T21:53:32.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:32 smithi121 ceph-mon[24681]: pgmap v70: 129 pgs: 17 unknown, 9 creating+peering, 103 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T21:53:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:32 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-13T21:53:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:32 smithi121 ceph-mon[24681]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T21:53:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:32 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-13T21:53:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:32 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:32 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:32.955 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[23730]: pgmap v70: 129 pgs: 17 unknown, 9 creating+peering, 103 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[23730]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[28528]: pgmap v70: 129 pgs: 17 unknown, 9 creating+peering, 103 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-13T21:53:32.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[28528]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T21:53:32.957 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-13T21:53:32.957 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:32.957 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:32 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:33.366 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-09-13T21:53:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:33 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-13T21:53:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:33 smithi121 ceph-mon[24681]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T21:53:33.910 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:33 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-13T21:53:33.910 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:33 smithi114 ceph-mon[28528]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T21:53:33.910 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:33 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2605693596' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-13T21:53:33.910 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:33 smithi114 ceph-mon[23730]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T21:53:34.734 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:34 smithi114 ceph-mon[23730]: pgmap v73: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-13T21:53:34.734 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:34 smithi114 ceph-mon[28528]: pgmap v73: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-13T21:53:34.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:34 smithi121 ceph-mon[24681]: pgmap v73: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-13T21:53:35.201 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled rgw.foo update... 2024-09-13T21:53:35.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:35] "GET /metrics HTTP/1.1" 200 196823 "" "Prometheus/2.33.4" 2024-09-13T21:53:35.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-13T21:53:35.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-09-13T21:53:35.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:35.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:53:35.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-13T21:53:35.932 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:35.933 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:35.953 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph osd pool create foo' 2024-09-13T21:53:36.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: pgmap v74: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 185 B/s rd, 370 B/s wr, 1 op/s 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: from='client.14703 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: Saving service rgw.foo spec with placement count:2 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: Saving service rgw.foo spec with placement count:2 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: Deploying daemon rgw.foo.smithi114.vrzgda on smithi114 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[23730]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T21:53:36.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: pgmap v74: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 185 B/s rd, 370 B/s wr, 1 op/s 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: from='client.14703 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: Saving service rgw.foo spec with placement count:2 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: Saving service rgw.foo spec with placement count:2 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: Deploying daemon rgw.foo.smithi114.vrzgda on smithi114 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T21:53:36.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:36 smithi114 ceph-mon[28528]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T21:53:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: pgmap v74: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 185 B/s rd, 370 B/s wr, 1 op/s 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: from='client.14703 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: Saving service rgw.foo spec with placement count:2 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: Saving service rgw.foo spec with placement count:2 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: Deploying daemon rgw.foo.smithi114.vrzgda on smithi114 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T21:53:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:36 smithi121 ceph-mon[24681]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T21:53:37.767 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:37 smithi114 ceph-mon[23730]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T21:53:37.767 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:37 smithi114 ceph-mon[28528]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T21:53:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:37 smithi121 ceph-mon[24681]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T21:53:38.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:53:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:53:38.740 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:38 smithi114 ceph-mon[23730]: pgmap v77: 129 pgs: 4 peering, 125 active+clean; 580 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T21:53:38.741 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:38 smithi114 ceph-mon[28528]: pgmap v77: 129 pgs: 4 peering, 125 active+clean; 580 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T21:53:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:38 smithi121 ceph-mon[24681]: pgmap v77: 129 pgs: 4 peering, 125 active+clean; 580 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T21:53:39.664 INFO:teuthology.orchestra.run.smithi114.stderr:pool 'foo' created 2024-09-13T21:53:39.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:39 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/4078518751' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T21:53:39.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:39 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T21:53:39.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:39 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:39.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:39 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:39.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:39 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:53:39.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:39 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:39.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:39 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:40.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4078518751' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T21:53:40.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T21:53:40.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:40.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:40.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:53:40.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4078518751' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:40.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:39 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:40.400 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'rbd pool init foo' 2024-09-13T21:53:40.664 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[23730]: pgmap v78: 129 pgs: 4 peering, 125 active+clean; 580 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 786 B/s rd, 943 B/s wr, 2 op/s 2024-09-13T21:53:40.665 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[23730]: Deploying daemon rgw.foo.smithi121.ihcnui on smithi121 2024-09-13T21:53:40.665 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-13T21:53:40.665 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[23730]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T21:53:40.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:40 smithi121 ceph-mon[24681]: pgmap v78: 129 pgs: 4 peering, 125 active+clean; 580 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 786 B/s rd, 943 B/s wr, 2 op/s 2024-09-13T21:53:40.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:40 smithi121 ceph-mon[24681]: Deploying daemon rgw.foo.smithi121.ihcnui on smithi121 2024-09-13T21:53:40.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:40 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-13T21:53:40.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:40 smithi121 ceph-mon[24681]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T21:53:40.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[28528]: pgmap v78: 129 pgs: 4 peering, 125 active+clean; 580 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 786 B/s rd, 943 B/s wr, 2 op/s 2024-09-13T21:53:40.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[28528]: Deploying daemon rgw.foo.smithi121.ihcnui on smithi121 2024-09-13T21:53:40.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-13T21:53:40.962 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:40 smithi114 ceph-mon[28528]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T21:53:41.958 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:41 smithi114 ceph-mon[23730]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T21:53:41.959 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:41 smithi114 ceph-mon[28528]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T21:53:42.022 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:41 smithi121 ceph-mon[24681]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T21:53:42.882 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: pgmap v81: 161 pgs: 8 unknown, 153 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 5.5 KiB/s wr, 146 op/s; 142 B/s, 0 objects/s recovering 2024-09-13T21:53:42.882 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T21:53:42.882 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.882 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:42.882 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:42.882 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: Checking dashboard <-> RGW credentials 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/857975016' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: pgmap v81: 161 pgs: 8 unknown, 153 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 5.5 KiB/s wr, 146 op/s; 142 B/s, 0 objects/s recovering 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.883 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: Checking dashboard <-> RGW credentials 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/857975016' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T21:53:42.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:42 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:43.058 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: pgmap v81: 161 pgs: 8 unknown, 153 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 5.5 KiB/s wr, 146 op/s; 142 B/s, 0 objects/s recovering 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: Checking dashboard <-> RGW credentials 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/857975016' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-13T21:53:43.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:42 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:43 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:43 smithi121 ceph-mon[24681]: Checking dashboard <-> RGW credentials 2024-09-13T21:53:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:43 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:43 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/857975016' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-13T21:53:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:43 smithi121 ceph-mon[24681]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T21:53:44.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[23730]: Checking dashboard <-> RGW credentials 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/857975016' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[23730]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[28528]: Checking dashboard <-> RGW credentials 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/857975016' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-13T21:53:44.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:43 smithi114 ceph-mon[28528]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T21:53:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:44 smithi121 ceph-mon[24681]: pgmap v83: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.7 KiB/s wr, 187 op/s; 130 B/s, 0 objects/s recovering 2024-09-13T21:53:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:44 smithi121 ceph-mon[24681]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T21:53:45.029 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:44 smithi114 ceph-mon[23730]: pgmap v83: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.7 KiB/s wr, 187 op/s; 130 B/s, 0 objects/s recovering 2024-09-13T21:53:45.029 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:44 smithi114 ceph-mon[23730]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T21:53:45.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:44 smithi114 ceph-mon[28528]: pgmap v83: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.7 KiB/s wr, 187 op/s; 130 B/s, 0 objects/s recovering 2024-09-13T21:53:45.029 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:44 smithi114 ceph-mon[28528]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T21:53:45.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:45] "GET /metrics HTTP/1.1" 200 196823 "" "Prometheus/2.33.4" 2024-09-13T21:53:45.789 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:53:45] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:46.117 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch apply iscsi foo u p' 2024-09-13T21:53:46.439 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[23730]: pgmap v86: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 7.6 KiB/s wr, 251 op/s; 175 B/s, 0 objects/s recovering 2024-09-13T21:53:46.439 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:46.439 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[23730]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T21:53:46.439 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:46.440 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[28528]: pgmap v86: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 7.6 KiB/s wr, 251 op/s; 175 B/s, 0 objects/s recovering 2024-09-13T21:53:46.440 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:46.440 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[28528]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T21:53:46.440 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:46.511 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:46 smithi121 ceph-mon[24681]: pgmap v86: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 7.6 KiB/s wr, 251 op/s; 175 B/s, 0 objects/s recovering 2024-09-13T21:53:46.511 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:46.511 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:46 smithi121 ceph-mon[24681]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T21:53:46.512 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.147 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled iscsi.foo update... 2024-09-13T21:53:48.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:53:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:53:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:48 smithi121 ceph-mon[24681]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 939 B/s wr, 198 op/s 2024-09-13T21:53:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:48 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:53:48] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:48.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[28528]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 939 B/s wr, 198 op/s 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[23730]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 939 B/s wr, 198 op/s 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:48.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:48 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:48.981 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T21:53:49.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:49 smithi114 ceph-mon[23730]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:49.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:49 smithi114 ceph-mon[23730]: Saving service iscsi.foo spec with placement count:1 2024-09-13T21:53:49.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:49 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:49.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:49 smithi114 ceph-mon[28528]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:49.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:49 smithi114 ceph-mon[28528]: Saving service iscsi.foo spec with placement count:1 2024-09-13T21:53:49.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:49 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:49.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:49 smithi121 ceph-mon[24681]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:49.472 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:49 smithi121 ceph-mon[24681]: Saving service iscsi.foo spec with placement count:1 2024-09-13T21:53:49.472 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:49 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:49.719 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:53:49] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:50.629 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[23730]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s rd, 853 B/s wr, 180 op/s 2024-09-13T21:53:50.629 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.629 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.629 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.629 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[28528]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s rd, 853 B/s wr, 180 op/s 2024-09-13T21:53:50.629 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.630 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.630 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:50 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:50 smithi121 ceph-mon[24681]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s rd, 853 B/s wr, 180 op/s 2024-09-13T21:53:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:50 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:51.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:51 smithi114 ceph-mon[23730]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T21:53:51.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:51 smithi114 ceph-mon[28528]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T21:53:51.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:53:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:51.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:51 smithi121 ceph-mon[24681]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T21:53:52.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 739 B/s wr, 156 op/s 2024-09-13T21:53:52.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:52.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:52.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: Checking pool "foo" exists for service iscsi.foo 2024-09-13T21:53:52.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T21:53:52.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-13T21:53:52.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:52.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[23730]: Deploying daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T21:53:52.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 739 B/s wr, 156 op/s 2024-09-13T21:53:52.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:52.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:52.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: Checking pool "foo" exists for service iscsi.foo 2024-09-13T21:53:52.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T21:53:52.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-13T21:53:52.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:52.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:52 smithi114 ceph-mon[28528]: Deploying daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T21:53:52.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 739 B/s wr, 156 op/s 2024-09-13T21:53:52.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: Checking pool "foo" exists for service iscsi.foo 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:52.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:52 smithi121 ceph-mon[24681]: Deploying daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 639 B/s wr, 135 op/s 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: mgrmap e20: y(active, since 2m), standbys: x 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi114"}]: dispatch 2024-09-13T21:53:54.597 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 639 B/s wr, 135 op/s 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: mgrmap e20: y(active, since 2m), standbys: x 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T21:53:54.598 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.599 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi114"}]: dispatch 2024-09-13T21:53:54.599 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.599 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:54.599 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:54.599 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:54 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 639 B/s wr, 135 op/s 2024-09-13T21:53:54.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: mgrmap e20: y(active, since 2m), standbys: x 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi114"}]: dispatch 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:54.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:54 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:55.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:55] "GET /metrics HTTP/1.1" 200 208087 "" "Prometheus/2.33.4" 2024-09-13T21:53:55.809 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:55.809 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T21:53:55.809 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: Adding iSCSI gateway http://:@172.21.15.114:5000 to Dashboard 2024-09-13T21:53:55.810 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T21:53:55.810 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi114"}]: dispatch 2024-09-13T21:53:55.810 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:55.810 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2989840851' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T21:53:55.810 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:55.810 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2389394836' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3483901434"}]: dispatch 2024-09-13T21:53:55.811 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:55.811 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T21:53:55.811 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: Adding iSCSI gateway http://:@172.21.15.114:5000 to Dashboard 2024-09-13T21:53:55.811 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T21:53:55.811 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi114"}]: dispatch 2024-09-13T21:53:55.812 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:55.812 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2989840851' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T21:53:55.812 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:55.812 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:55 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2389394836' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3483901434"}]: dispatch 2024-09-13T21:53:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T21:53:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: Adding iSCSI gateway http://:@172.21.15.114:5000 to Dashboard 2024-09-13T21:53:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T21:53:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi114"}]: dispatch 2024-09-13T21:53:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T21:53:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2989840851' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T21:53:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:55 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2389394836' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3483901434"}]: dispatch 2024-09-13T21:53:56.739 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[23730]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 515 B/s wr, 108 op/s 2024-09-13T21:53:56.739 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2389394836' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3483901434"}]': finished 2024-09-13T21:53:56.739 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[23730]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T21:53:56.739 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1748974604' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]: dispatch 2024-09-13T21:53:56.740 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]: dispatch 2024-09-13T21:53:56.740 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[28528]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 515 B/s wr, 108 op/s 2024-09-13T21:53:56.740 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2389394836' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3483901434"}]': finished 2024-09-13T21:53:56.740 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[28528]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T21:53:56.740 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1748974604' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]: dispatch 2024-09-13T21:53:56.740 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:56 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]: dispatch 2024-09-13T21:53:56.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:56 smithi121 ceph-mon[24681]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 515 B/s wr, 108 op/s 2024-09-13T21:53:56.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:56 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2389394836' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3483901434"}]': finished 2024-09-13T21:53:56.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:56 smithi121 ceph-mon[24681]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T21:53:56.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:56 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1748974604' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]: dispatch 2024-09-13T21:53:56.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:56 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]: dispatch 2024-09-13T21:53:58.197 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:53:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:53:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:53:58.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[28528]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 1 op/s 2024-09-13T21:53:58.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]': finished 2024-09-13T21:53:58.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[28528]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T21:53:58.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3462924903' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1462675355"}]: dispatch 2024-09-13T21:53:58.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[23730]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 1 op/s 2024-09-13T21:53:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]': finished 2024-09-13T21:53:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[23730]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T21:53:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3462924903' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1462675355"}]: dispatch 2024-09-13T21:53:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:58.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:53:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:53:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:53:58.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:58 smithi121 ceph-mon[24681]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 1 op/s 2024-09-13T21:53:58.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:58 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4155242912"}]': finished 2024-09-13T21:53:58.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:58 smithi121 ceph-mon[24681]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T21:53:58.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:58 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3462924903' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1462675355"}]: dispatch 2024-09-13T21:53:58.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:53:59.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3462924903' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1462675355"}]': finished 2024-09-13T21:53:59.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[23730]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T21:53:59.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1846861238' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]: dispatch 2024-09-13T21:53:59.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]: dispatch 2024-09-13T21:53:59.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3462924903' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1462675355"}]': finished 2024-09-13T21:53:59.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[28528]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T21:53:59.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1846861238' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]: dispatch 2024-09-13T21:53:59.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:53:59 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]: dispatch 2024-09-13T21:53:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:59 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3462924903' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1462675355"}]': finished 2024-09-13T21:53:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:59 smithi121 ceph-mon[24681]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T21:53:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:59 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1846861238' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]: dispatch 2024-09-13T21:53:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:53:59 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]: dispatch 2024-09-13T21:54:00.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[28528]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 1 op/s 2024-09-13T21:54:00.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]': finished 2024-09-13T21:54:00.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[28528]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T21:54:00.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/906424164' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/408371097"}]: dispatch 2024-09-13T21:54:00.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[23730]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 1 op/s 2024-09-13T21:54:00.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]': finished 2024-09-13T21:54:00.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[23730]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T21:54:00.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/906424164' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/408371097"}]: dispatch 2024-09-13T21:54:00.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:00 smithi121 ceph-mon[24681]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 1 op/s 2024-09-13T21:54:00.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:00 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/408371097"}]': finished 2024-09-13T21:54:00.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:00 smithi121 ceph-mon[24681]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T21:54:00.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:00 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/906424164' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/408371097"}]: dispatch 2024-09-13T21:54:01.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:00 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:54:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/906424164' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/408371097"}]': finished 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3123413548' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1573161012"}]: dispatch 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:54:01.543 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/906424164' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/408371097"}]': finished 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3123413548' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1573161012"}]: dispatch 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:54:01.544 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/906424164' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/408371097"}]': finished 2024-09-13T21:54:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T21:54:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3123413548' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1573161012"}]: dispatch 2024-09-13T21:54:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:54:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:54:01.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:02.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[23730]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:02.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3123413548' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1573161012"}]': finished 2024-09-13T21:54:02.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[23730]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T21:54:02.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4003125837' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/4206988749"}]: dispatch 2024-09-13T21:54:02.800 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[28528]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:02.800 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3123413548' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1573161012"}]': finished 2024-09-13T21:54:02.800 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[28528]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T21:54:02.801 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:02 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4003125837' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/4206988749"}]: dispatch 2024-09-13T21:54:02.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:02 smithi121 ceph-mon[24681]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:02.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:02 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3123413548' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1573161012"}]': finished 2024-09-13T21:54:02.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:02 smithi121 ceph-mon[24681]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T21:54:02.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:02 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/4003125837' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/4206988749"}]: dispatch 2024-09-13T21:54:03.785 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:54:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:54:03.786 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:54:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:54:03.786 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:54:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:54:04.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4003125837' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/4206988749"}]': finished 2024-09-13T21:54:04.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[23730]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T21:54:04.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1212790371' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]: dispatch 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]: dispatch 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4003125837' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/4206988749"}]': finished 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[28528]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1212790371' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]: dispatch 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]: dispatch 2024-09-13T21:54:04.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:04.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:03 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:03 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/4003125837' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/4206988749"}]': finished 2024-09-13T21:54:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:03 smithi121 ceph-mon[24681]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T21:54:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:03 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1212790371' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]: dispatch 2024-09-13T21:54:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:03 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]: dispatch 2024-09-13T21:54:04.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:03 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:04.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:03 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:05.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[28528]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:05.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]': finished 2024-09-13T21:54:05.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[28528]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T21:54:05.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/642659064' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/4206988749"}]: dispatch 2024-09-13T21:54:05.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[23730]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:05.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]': finished 2024-09-13T21:54:05.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[23730]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T21:54:05.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:04 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/642659064' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/4206988749"}]: dispatch 2024-09-13T21:54:05.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:04 smithi121 ceph-mon[24681]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:05.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:04 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2424716870"}]': finished 2024-09-13T21:54:05.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:04 smithi121 ceph-mon[24681]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T21:54:05.114 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:04 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/642659064' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/4206988749"}]: dispatch 2024-09-13T21:54:05.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:05] "GET /metrics HTTP/1.1" 200 208093 "" "Prometheus/2.33.4" 2024-09-13T21:54:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/642659064' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/4206988749"}]': finished 2024-09-13T21:54:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[23730]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T21:54:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3817313314' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3146194524"}]: dispatch 2024-09-13T21:54:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/642659064' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/4206988749"}]': finished 2024-09-13T21:54:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[28528]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T21:54:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:05 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3817313314' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3146194524"}]: dispatch 2024-09-13T21:54:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:05 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:05 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/642659064' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/4206988749"}]': finished 2024-09-13T21:54:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:05 smithi121 ceph-mon[24681]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T21:54:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:05 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3817313314' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3146194524"}]: dispatch 2024-09-13T21:54:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[28528]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:54:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3817313314' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3146194524"}]': finished 2024-09-13T21:54:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[28528]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T21:54:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/707339987' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2980892939"}]: dispatch 2024-09-13T21:54:07.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[23730]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:54:07.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3817313314' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3146194524"}]': finished 2024-09-13T21:54:07.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[23730]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T21:54:07.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:06 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/707339987' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2980892939"}]: dispatch 2024-09-13T21:54:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:06 smithi121 ceph-mon[24681]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:54:07.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:06 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3817313314' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3146194524"}]': finished 2024-09-13T21:54:07.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:06 smithi121 ceph-mon[24681]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T21:54:07.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:06 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/707339987' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2980892939"}]: dispatch 2024-09-13T21:54:08.048 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:07 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/707339987' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2980892939"}]': finished 2024-09-13T21:54:08.048 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:07 smithi121 ceph-mon[24681]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T21:54:08.048 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:07 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/770109503' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]: dispatch 2024-09-13T21:54:08.048 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:07 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]: dispatch 2024-09-13T21:54:08.067 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/707339987' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2980892939"}]': finished 2024-09-13T21:54:08.067 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[23730]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T21:54:08.067 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/770109503' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]: dispatch 2024-09-13T21:54:08.067 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]: dispatch 2024-09-13T21:54:08.067 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/707339987' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2980892939"}]': finished 2024-09-13T21:54:08.068 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[28528]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T21:54:08.068 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/770109503' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]: dispatch 2024-09-13T21:54:08.068 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:07 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]: dispatch 2024-09-13T21:54:08.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:54:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:54:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:08 smithi121 ceph-mon[24681]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:08 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]': finished 2024-09-13T21:54:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:08 smithi121 ceph-mon[24681]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T21:54:09.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:08 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/4000809064' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/2918421085"}]: dispatch 2024-09-13T21:54:09.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[23730]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]': finished 2024-09-13T21:54:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[23730]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T21:54:09.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4000809064' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/2918421085"}]: dispatch 2024-09-13T21:54:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[28528]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/2918421085"}]': finished 2024-09-13T21:54:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[28528]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T21:54:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:08 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4000809064' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/2918421085"}]: dispatch 2024-09-13T21:54:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:09 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/4000809064' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/2918421085"}]': finished 2024-09-13T21:54:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:09 smithi121 ceph-mon[24681]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T21:54:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:09 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3691149824' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]: dispatch 2024-09-13T21:54:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:09 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]: dispatch 2024-09-13T21:54:10.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4000809064' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/2918421085"}]': finished 2024-09-13T21:54:10.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[23730]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T21:54:10.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3691149824' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]: dispatch 2024-09-13T21:54:10.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]: dispatch 2024-09-13T21:54:10.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4000809064' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/2918421085"}]': finished 2024-09-13T21:54:10.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[28528]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T21:54:10.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3691149824' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]: dispatch 2024-09-13T21:54:10.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:09 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]: dispatch 2024-09-13T21:54:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:10 smithi121 ceph-mon[24681]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:10 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]': finished 2024-09-13T21:54:11.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:10 smithi121 ceph-mon[24681]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T21:54:11.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:10 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3469178239' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]: dispatch 2024-09-13T21:54:11.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:10 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]: dispatch 2024-09-13T21:54:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[23730]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]': finished 2024-09-13T21:54:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[23730]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T21:54:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3469178239' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]: dispatch 2024-09-13T21:54:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]: dispatch 2024-09-13T21:54:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[28528]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/26654585"}]': finished 2024-09-13T21:54:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[28528]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T21:54:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3469178239' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]: dispatch 2024-09-13T21:54:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:10 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]: dispatch 2024-09-13T21:54:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:11 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]': finished 2024-09-13T21:54:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:11 smithi121 ceph-mon[24681]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T21:54:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:11 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2798132606' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/115794601"}]: dispatch 2024-09-13T21:54:12.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:11 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]': finished 2024-09-13T21:54:12.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:11 smithi114 ceph-mon[23730]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T21:54:12.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:11 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2798132606' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/115794601"}]: dispatch 2024-09-13T21:54:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:11 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/206701471"}]': finished 2024-09-13T21:54:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:11 smithi114 ceph-mon[28528]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T21:54:12.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:11 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2798132606' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/115794601"}]: dispatch 2024-09-13T21:54:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:12 smithi121 ceph-mon[24681]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:12 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2798132606' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/115794601"}]': finished 2024-09-13T21:54:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:12 smithi121 ceph-mon[24681]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T21:54:13.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:12 smithi114 ceph-mon[23730]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:13.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:12 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2798132606' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/115794601"}]': finished 2024-09-13T21:54:13.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:12 smithi114 ceph-mon[23730]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T21:54:13.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:12 smithi114 ceph-mon[28528]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:13.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:12 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2798132606' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/115794601"}]': finished 2024-09-13T21:54:13.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:12 smithi114 ceph-mon[28528]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T21:54:13.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:54:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:54:13.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:54:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:14 smithi114 ceph-mon[23730]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:14 smithi114 ceph-mon[28528]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:14 smithi121 ceph-mon[24681]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:15.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:15] "GET /metrics HTTP/1.1" 200 208093 "" "Prometheus/2.33.4" 2024-09-13T21:54:15.990 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:15 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:15.991 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:15 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:15.991 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:54:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:54:16.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:15 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:17.004 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:16 smithi114 ceph-mon[23730]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:17.004 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:16 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:17.004 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:16 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:17.004 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:16 smithi114 ceph-mon[28528]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:17.004 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:16 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:17.004 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:16 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:16 smithi121 ceph-mon[24681]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:16 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:16 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:18.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:54:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:54:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:18 smithi114 ceph-mon[23730]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T21:54:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:18 smithi114 ceph-mon[28528]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T21:54:19.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:18 smithi121 ceph-mon[24681]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T21:54:21.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:20 smithi114 ceph-mon[28528]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:20 smithi114 ceph-mon[23730]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:20 smithi121 ceph-mon[24681]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:23.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:22 smithi114 ceph-mon[28528]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T21:54:23.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:22 smithi114 ceph-mon[23730]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T21:54:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:22 smithi121 ceph-mon[24681]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T21:54:24.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:54:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:54:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:24 smithi114 ceph-mon[23730]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T21:54:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:24 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:24 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:25.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:24 smithi114 ceph-mon[28528]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T21:54:25.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:24 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:25.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:24 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:24 smithi121 ceph-mon[24681]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T21:54:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:24 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:24 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:25.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:25] "GET /metrics HTTP/1.1" 200 208120 "" "Prometheus/2.33.4" 2024-09-13T21:54:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:25 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:26.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:25 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:25 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:27.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:26 smithi114 ceph-mon[23730]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:27.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:26 smithi114 ceph-mon[28528]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:26 smithi121 ceph-mon[24681]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:28.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:54:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:54:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:28 smithi114 ceph-mon[23730]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:28 smithi114 ceph-mon[28528]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:29.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:28 smithi121 ceph-mon[24681]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:31.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:30 smithi114 ceph-mon[23730]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:31.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:30 smithi114 ceph-mon[28528]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:30 smithi121 ceph-mon[24681]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:33.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:32 smithi114 ceph-mon[23730]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:33.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:32 smithi114 ceph-mon[28528]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:33.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:32 smithi121 ceph-mon[24681]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:35.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:34 smithi114 ceph-mon[23730]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:35.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:34 smithi114 ceph-mon[28528]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:34 smithi121 ceph-mon[24681]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:35.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:35] "GET /metrics HTTP/1.1" 200 208071 "" "Prometheus/2.33.4" 2024-09-13T21:54:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:35 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:54:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:54:36.274 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:54:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:54:36.275 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:35 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:36.275 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:54:36.275 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:54:36.275 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:35 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:36.275 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:54:36.275 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:54:37.278 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:37 smithi114 ceph-mon[28528]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:37.278 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:37 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:37.278 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:37 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:37.278 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:37 smithi114 ceph-mon[23730]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:37.279 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:37 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:37.279 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:37 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:37 smithi121 ceph-mon[24681]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:37 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:37 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:38.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:54:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:54:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:39 smithi114 ceph-mon[23730]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:39 smithi114 ceph-mon[28528]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:39 smithi121 ceph-mon[24681]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:41 smithi114 ceph-mon[23730]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:41 smithi114 ceph-mon[28528]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:41 smithi121 ceph-mon[24681]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:43.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:43 smithi114 ceph-mon[23730]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:43.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:43 smithi114 ceph-mon[28528]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:43.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:43 smithi121 ceph-mon[24681]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:44.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:54:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:54:45.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:44 smithi114 ceph-mon[23730]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:45.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:45.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:45.235 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:44 smithi114 ceph-mon[28528]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:45.235 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:44 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:45.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:44 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:44 smithi121 ceph-mon[24681]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:45.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:45] "GET /metrics HTTP/1.1" 200 208071 "" "Prometheus/2.33.4" 2024-09-13T21:54:46.267 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:45 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:46.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:45 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:46.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:45 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:47.272 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:46 smithi121 ceph-mon[24681]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:47.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:46 smithi114 ceph-mon[23730]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:47.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:46 smithi114 ceph-mon[28528]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:48.592 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:54:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:54:49.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:48 smithi114 ceph-mon[23730]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:48 smithi114 ceph-mon[28528]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:48 smithi121 ceph-mon[24681]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:50 smithi114 ceph-mon[23730]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:51.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:50 smithi114 ceph-mon[28528]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:50 smithi121 ceph-mon[24681]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:53.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:52 smithi114 ceph-mon[23730]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:52 smithi114 ceph-mon[28528]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:52 smithi121 ceph-mon[24681]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:55.173 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:54 smithi121 ceph-mon[24681]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:55.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:54 smithi114 ceph-mon[23730]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:55.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:54 smithi114 ceph-mon[28528]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:55.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:55] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-09-13T21:54:56.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:55 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:56.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:55 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:56.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:54:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:54:56] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:54:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:55 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:54:57.036 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:56 smithi114 ceph-mon[23730]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:57.036 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:57.037 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:57.037 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:56 smithi114 ceph-mon[28528]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:57.037 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:57.037 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:56 smithi121 ceph-mon[24681]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:54:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:54:58.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:54:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:54:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:54:59.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:54:58 smithi114 ceph-mon[23730]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:59.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:54:58 smithi114 ceph-mon[28528]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:54:59.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:54:58 smithi121 ceph-mon[24681]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[23730]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:55:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:55:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[28528]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:55:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:55:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:00 smithi121 ceph-mon[24681]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:01.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:55:01.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:55:01.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:03.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:02 smithi114 ceph-mon[23730]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:03.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:02 smithi114 ceph-mon[28528]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:03.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:02 smithi121 ceph-mon[24681]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:04.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:55:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:55:05.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:05 smithi114 ceph-mon[28528]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:05.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:05.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:05.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:05 smithi114 ceph-mon[23730]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:05.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:05.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:05.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:05] "GET /metrics HTTP/1.1" 200 208031 "" "Prometheus/2.33.4" 2024-09-13T21:55:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:05 smithi121 ceph-mon[24681]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:06.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:06 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:06.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:06 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:06.328 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:06 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:07 smithi121 ceph-mon[24681]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:07.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:07 smithi114 ceph-mon[23730]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:07.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:07 smithi114 ceph-mon[28528]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:08.460 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:55:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:55:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:09 smithi121 ceph-mon[24681]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:09.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:09 smithi114 ceph-mon[23730]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:09.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:09 smithi114 ceph-mon[28528]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:11 smithi114 ceph-mon[23730]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:11 smithi114 ceph-mon[28528]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:11.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:11 smithi121 ceph-mon[24681]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:13.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:13 smithi114 ceph-mon[23730]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:13.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:13 smithi114 ceph-mon[28528]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:13 smithi121 ceph-mon[24681]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:15.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:15 smithi114 ceph-mon[23730]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:15 smithi114 ceph-mon[28528]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:15.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:15] "GET /metrics HTTP/1.1" 200 208031 "" "Prometheus/2.33.4" 2024-09-13T21:55:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:15 smithi121 ceph-mon[24681]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:16.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:16 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:16.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:16 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:55:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:55:16.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:16 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:17.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:17 smithi114 ceph-mon[23730]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:17.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:17 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:17.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:17 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:17 smithi114 ceph-mon[28528]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:17 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:17 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:17 smithi121 ceph-mon[24681]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:17 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:17 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:18.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:55:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:55:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:19 smithi114 ceph-mon[23730]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:19 smithi114 ceph-mon[28528]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:19.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:19 smithi121 ceph-mon[24681]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:21 smithi114 ceph-mon[23730]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:21 smithi114 ceph-mon[28528]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:21 smithi121 ceph-mon[24681]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:23.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:23 smithi114 ceph-mon[23730]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:23.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:23 smithi114 ceph-mon[28528]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:23 smithi121 ceph-mon[24681]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:24.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:55:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:55:25.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:25 smithi114 ceph-mon[28528]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:25 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:25 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:25.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:25 smithi114 ceph-mon[23730]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:25.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:25 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:25.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:25 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:25.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:25] "GET /metrics HTTP/1.1" 200 208031 "" "Prometheus/2.33.4" 2024-09-13T21:55:25.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:25 smithi121 ceph-mon[24681]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:25.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:25 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:25.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:25 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:26.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:26 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:26.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:26 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:26 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:27.217 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:27 smithi121 ceph-mon[24681]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:27.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:27 smithi114 ceph-mon[23730]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:27 smithi114 ceph-mon[28528]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:28.429 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:55:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:55:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:29 smithi114 ceph-mon[23730]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:29.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:29 smithi114 ceph-mon[28528]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:29 smithi121 ceph-mon[24681]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:31.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:31 smithi114 ceph-mon[23730]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:31 smithi114 ceph-mon[28528]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:31.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:31 smithi121 ceph-mon[24681]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:33.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:33 smithi114 ceph-mon[23730]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:33.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:33 smithi114 ceph-mon[28528]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:33 smithi121 ceph-mon[24681]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:35.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:35 smithi114 ceph-mon[28528]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:35.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:35 smithi114 ceph-mon[23730]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:35.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:35] "GET /metrics HTTP/1.1" 200 208047 "" "Prometheus/2.33.4" 2024-09-13T21:55:35.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:35 smithi121 ceph-mon[24681]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:36 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:55:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:55:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:36 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:55:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:55:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:36 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:55:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:55:36.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:55:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:55:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:37 smithi114 ceph-mon[23730]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:37 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:37 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:37 smithi114 ceph-mon[28528]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:37 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:37 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:37 smithi121 ceph-mon[24681]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:37 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:37 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:38.609 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:55:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:55:39.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:39 smithi114 ceph-mon[23730]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:39.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:39 smithi114 ceph-mon[28528]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:39 smithi121 ceph-mon[24681]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:41 smithi114 ceph-mon[23730]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:41 smithi114 ceph-mon[28528]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:41 smithi121 ceph-mon[24681]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:43 smithi121 ceph-mon[24681]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:43.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:43 smithi114 ceph-mon[23730]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:43.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:43 smithi114 ceph-mon[28528]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:44.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:44 smithi114 ceph-mon[23730]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:44 smithi114 ceph-mon[28528]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:44.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:55:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:55:44.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:44 smithi121 ceph-mon[24681]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:45 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:45 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:45 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:45] "GET /metrics HTTP/1.1" 200 208047 "" "Prometheus/2.33.4" 2024-09-13T21:55:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:45 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:45.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:45 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:45.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:45 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:45.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:45 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:46.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:46 smithi114 ceph-mon[23730]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:46 smithi114 ceph-mon[28528]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:46 smithi121 ceph-mon[24681]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:48.295 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:55:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:55:48.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:48 smithi114 ceph-mon[23730]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:48.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:48 smithi114 ceph-mon[28528]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:48 smithi121 ceph-mon[24681]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:50.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:50 smithi114 ceph-mon[23730]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:50.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:50 smithi114 ceph-mon[28528]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:50 smithi121 ceph-mon[24681]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:52.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:52 smithi114 ceph-mon[23730]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:52.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:52 smithi114 ceph-mon[28528]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:52.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:52 smithi121 ceph-mon[24681]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:54.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:54 smithi114 ceph-mon[23730]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:54.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:54 smithi114 ceph-mon[28528]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:54.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:54 smithi121 ceph-mon[24681]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:55.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:55] "GET /metrics HTTP/1.1" 200 208036 "" "Prometheus/2.33.4" 2024-09-13T21:55:55.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:55 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:55 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:55 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:55:56.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:55:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:55:56] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:55:56.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:56 smithi114 ceph-mon[23730]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:56.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:56.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:56.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:56 smithi114 ceph-mon[28528]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:56.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:56.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:56.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:56 smithi121 ceph-mon[24681]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:55:56.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:56.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:55:58.568 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:55:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:55:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:55:58.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:55:58 smithi114 ceph-mon[23730]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:55:58 smithi114 ceph-mon[28528]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:55:58.838 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:55:58 smithi121 ceph-mon[24681]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:00.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:00 smithi114 ceph-mon[23730]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:00.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:00 smithi114 ceph-mon[28528]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:00 smithi121 ceph-mon[24681]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:01.639 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:56:01.639 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:56:01.639 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:01 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:01.639 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:56:01.639 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:56:01.640 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:01 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:01.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:56:01.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:56:01.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:01 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:03.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:02 smithi114 ceph-mon[23730]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:03.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:02 smithi114 ceph-mon[28528]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:02 smithi121 ceph-mon[24681]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:04.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:56:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:56:05.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:04 smithi114 ceph-mon[23730]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:05.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:04 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:05.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:04 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:05.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:04 smithi114 ceph-mon[28528]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:05.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:04 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:05.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:04 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:05.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:04 smithi121 ceph-mon[24681]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:05.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:04 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:05.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:04 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:05.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:05] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-09-13T21:56:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:05 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:05 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:05 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:06 smithi114 ceph-mon[23730]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:06 smithi114 ceph-mon[28528]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:06 smithi121 ceph-mon[24681]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:08.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:56:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:56:08.990 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:08 smithi121 ceph-mon[24681]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:08 smithi114 ceph-mon[23730]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:08 smithi114 ceph-mon[28528]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:10 smithi114 ceph-mon[23730]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:10 smithi114 ceph-mon[28528]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:10 smithi121 ceph-mon[24681]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:13.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:12 smithi114 ceph-mon[28528]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:13.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:12 smithi114 ceph-mon[23730]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:12 smithi121 ceph-mon[24681]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:14 smithi114 ceph-mon[23730]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:14 smithi114 ceph-mon[28528]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:14 smithi121 ceph-mon[24681]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:15.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:15] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-09-13T21:56:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:15 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:16.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:15 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:16.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:15 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:16.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:56:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:56:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:16 smithi121 ceph-mon[24681]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:16 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:16 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:17.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:16 smithi114 ceph-mon[23730]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:16 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:16 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:16 smithi114 ceph-mon[28528]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:16 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:16 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:18.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:56:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:56:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:18 smithi121 ceph-mon[24681]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:18 smithi114 ceph-mon[23730]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:18 smithi114 ceph-mon[28528]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:20 smithi121 ceph-mon[24681]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:21.122 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:20 smithi114 ceph-mon[23730]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:21.122 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:20 smithi114 ceph-mon[28528]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:22 smithi114 ceph-mon[23730]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:22 smithi114 ceph-mon[28528]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:22 smithi121 ceph-mon[24681]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:24.741 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:56:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:56:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:24 smithi114 ceph-mon[23730]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:24 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:24 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:25.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:24 smithi114 ceph-mon[28528]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:25.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:24 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:25.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:24 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:24 smithi121 ceph-mon[24681]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:24 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:24 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:25.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:25] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-09-13T21:56:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:25 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:26.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:25 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:26.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:25 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:26 smithi121 ceph-mon[24681]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:27.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:26 smithi114 ceph-mon[23730]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:26 smithi114 ceph-mon[28528]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:28.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:56:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:56:29.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:28 smithi121 ceph-mon[24681]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:28 smithi114 ceph-mon[23730]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:29.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:28 smithi114 ceph-mon[28528]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:30 smithi121 ceph-mon[24681]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:31.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:30 smithi114 ceph-mon[23730]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:30 smithi114 ceph-mon[28528]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:33.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:32 smithi121 ceph-mon[24681]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:33.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:32 smithi114 ceph-mon[23730]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:33.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:32 smithi114 ceph-mon[28528]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:35.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:34 smithi114 ceph-mon[23730]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:35.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:34 smithi114 ceph-mon[28528]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:34 smithi121 ceph-mon[24681]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:35.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:35] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-09-13T21:56:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:35 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:56:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:56:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:35 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:56:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:56:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:35 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:56:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:56:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:36 smithi114 ceph-mon[23730]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:37.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:56:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:56:37.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:36 smithi114 ceph-mon[28528]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:37.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:37.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:36 smithi121 ceph-mon[24681]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:38.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:56:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:56:39.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:38 smithi121 ceph-mon[24681]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:39.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:38 smithi114 ceph-mon[23730]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:38 smithi114 ceph-mon[28528]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:41.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:40 smithi114 ceph-mon[23730]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:41.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:40 smithi114 ceph-mon[28528]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:40 smithi121 ceph-mon[24681]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:43.162 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:42 smithi114 ceph-mon[23730]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:43.163 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:42 smithi114 ceph-mon[28528]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:42 smithi121 ceph-mon[24681]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:44.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:56:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:56:45.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:44 smithi114 ceph-mon[23730]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:45.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:45.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:45.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:44 smithi114 ceph-mon[28528]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:45.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:44 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:45.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:44 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:44 smithi121 ceph-mon[24681]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:45.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:45] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-09-13T21:56:46.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:45 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:46.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:45 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:46.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:45 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:47.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:46 smithi114 ceph-mon[23730]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:47.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:46 smithi114 ceph-mon[28528]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:46 smithi121 ceph-mon[24681]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:56:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:56:49.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:48 smithi114 ceph-mon[23730]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:49.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:48 smithi114 ceph-mon[28528]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:48 smithi121 ceph-mon[24681]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:50 smithi114 ceph-mon[23730]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:50 smithi114 ceph-mon[28528]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:50 smithi121 ceph-mon[24681]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:51.457 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T21:56:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:52 smithi121 ceph-mon[24681]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:53.182 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:52 smithi114 ceph-mon[23730]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:53.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:52 smithi114 ceph-mon[28528]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:53.753 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T21:56:55.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:54 smithi114 ceph-mon[23730]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:55.068 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:54 smithi114 ceph-mon[28528]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:55.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:55] "GET /metrics HTTP/1.1" 200 207990 "" "Prometheus/2.33.4" 2024-09-13T21:56:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:54 smithi121 ceph-mon[24681]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:56.261 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:55 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:56.262 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:55 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:56.272 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:55 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:56:56.284 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T21:56:56.881 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:56:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:56:56] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:56:57.252 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:56 smithi114 ceph-mon[28528]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:57.252 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:57.252 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:56 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:57.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:56 smithi114 ceph-mon[23730]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:57.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:57.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:56 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:56 smithi121 ceph-mon[24681]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:56:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:56 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:56:58.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:56:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:56:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:56:58.944 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T21:56:59.240 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:56:58 smithi114 ceph-mon[23730]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:59.241 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:56:58 smithi114 ceph-mon[28528]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:56:59.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:56:58 smithi121 ceph-mon[24681]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:00.834 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:57:00.834 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 4s ago 9m - - 2024-09-13T21:57:00.834 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 16s ago 9m - - 2024-09-13T21:57:00.834 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (3m) 4s ago 4m 27.8M - ba2b418f427c 9bf04fad0c85 2024-09-13T21:57:00.834 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (4m) 16s ago 4m 51.5M - 8.3.5 dad864ee21e9 b8bdbef3d489 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (3m) 4s ago 3m 52.8M - 3.5 e1d6a67b021e d4d9db680881 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443 running (8m) 16s ago 8m 412M - 17.2.0 e1d6a67b021e 7f14340b5cbc 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:9283 running (10m) 4s ago 10m 462M - 17.2.0 e1d6a67b021e f0c0d3e1c024 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (10m) 4s ago 10m 49.6M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (9m) 16s ago 9m 45.3M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (9m) 4s ago 9m 39.6M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (4m) 4s ago 4m 15.7M - 1dbe0e931976 45aa09d329e6 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (4m) 16s ago 4m 16.9M - 1dbe0e931976 5452c9d8e4cb 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (8m) 4s ago 8m 52.1M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (8m) 4s ago 8m 51.8M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (7m) 4s ago 7m 45.6M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (7m) 4s ago 7m 46.8M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (7m) 16s ago 7m 47.5M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T21:57:00.835 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (6m) 16s ago 6m 48.5M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T21:57:00.836 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (6m) 16s ago 6m 45.5M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T21:57:00.836 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (5m) 16s ago 5m 47.2M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T21:57:00.836 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (3m) 16s ago 5m 67.0M - 514e6a882f6e baadd3f398d4 2024-09-13T21:57:00.836 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (3m) 4s ago 3m 69.9M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T21:57:00.836 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (3m) 16s ago 3m 70.9M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T21:57:01.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[23730]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:01.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:01.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:01.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:01.247 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[28528]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:01.247 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:01.247 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:01.248 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:00 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:00 smithi121 ceph-mon[24681]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:01.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:00 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:01.482 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T21:57:02.250 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:01 smithi114 ceph-mon[23730]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:02.251 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:01 smithi114 ceph-mon[28528]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:02.330 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:01 smithi121 ceph-mon[24681]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:03.260 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:02 smithi114 ceph-mon[23730]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:03.261 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:02 smithi114 ceph-mon[28528]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:03.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:02 smithi121 ceph-mon[24681]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:57:03.827 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T21:57:03.828 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T21:57:03.828 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:57:03.828 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T21:57:03.828 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T21:57:03.828 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-09-13T21:57:03.828 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T21:57:03.828 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T21:57:04.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:03 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2266464272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:57:04.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:03 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2266464272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:57:04.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:03 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2266464272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:57:04.592 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T21:57:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:57:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:57:05.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:04 smithi114 ceph-mon[23730]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:05.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:04 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:05.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:04 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:05.158 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:04 smithi114 ceph-mon[28528]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:05.158 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:04 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:05.158 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:04 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:04 smithi121 ceph-mon[24681]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:04 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:04 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:05.425 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:05] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-13T21:57:06.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:05 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:05 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:06.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:05 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:06.806 INFO:teuthology.orchestra.run.smithi114.stdout: cluster: 2024-09-13T21:57:06.806 INFO:teuthology.orchestra.run.smithi114.stdout: id: 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T21:57:06.806 INFO:teuthology.orchestra.run.smithi114.stdout: health: HEALTH_OK 2024-09-13T21:57:06.806 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: services: 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: mon: 3 daemons, quorum a,c,b (age 9m) 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: mgr: y(active, since 5m), standbys: x 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 6m) 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: data: 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: pools: 6 pools, 161 pgs 2024-09-13T21:57:06.807 INFO:teuthology.orchestra.run.smithi114.stdout: objects: 209 objects, 585 KiB 2024-09-13T21:57:06.808 INFO:teuthology.orchestra.run.smithi114.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:57:06.808 INFO:teuthology.orchestra.run.smithi114.stdout: pgs: 161 active+clean 2024-09-13T21:57:06.808 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:57:06.808 INFO:teuthology.orchestra.run.smithi114.stdout: io: 2024-09-13T21:57:06.808 INFO:teuthology.orchestra.run.smithi114.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-13T21:57:06.808 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T21:57:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:06 smithi114 ceph-mon[23730]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:06 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/470014933' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T21:57:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:06 smithi114 ceph-mon[28528]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:06 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/470014933' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T21:57:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:06 smithi121 ceph-mon[24681]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:06 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/470014933' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T21:57:07.903 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ls' 2024-09-13T21:57:08.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:57:09.214 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:08 smithi114 ceph-mon[23730]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:09.214 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:08 smithi114 ceph-mon[28528]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:09.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:08 smithi121 ceph-mon[24681]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:09.850 INFO:teuthology.orchestra.run.smithi114.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:agent 2/2 13s ago 5m * 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager ?:9093,9094 1/1 13s ago 5m smithi114=a;count:1 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:grafana ?:3000 1/1 5s ago 5m smithi121=a;count:1 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo 1/1 13s ago 3m count:1 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:mgr 2/2 13s ago 9m smithi114=y;smithi121=x;count:2 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:mon 3/3 13s ago 9m smithi114:172.21.15.114=a;smithi114:[v2:172.21.15.114:3301,v1:172.21.15.114:6790]=c;smithi121:172.21.15.121=b;count:3 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter ?:9100 2/2 13s ago 5m smithi114=a;smithi121=b;count:2 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:osd 8 13s ago - 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus ?:9095 1/1 5s ago 5m smithi121=a;count:1 2024-09-13T21:57:09.851 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo ?:8000 2/2 13s ago 3m count:2 2024-09-13T21:57:10.468 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T21:57:11.032 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:10 smithi114 ceph-mon[23730]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:11.032 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:10 smithi114 ceph-mon[23730]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:11.033 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:10 smithi114 ceph-mon[28528]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:11.033 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:10 smithi114 ceph-mon[28528]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:10 smithi121 ceph-mon[24681]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:10 smithi121 ceph-mon[24681]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:12.826 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled to redeploy mgr.x on host 'smithi121' 2024-09-13T21:57:13.111 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[23730]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/114145974' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[28528]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/114145974' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:13.112 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:12 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:12 smithi121 ceph-mon[24681]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:12 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/114145974' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:57:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:12 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:13.508 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps --refresh' 2024-09-13T21:57:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[23730]: from='client.24901 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[23730]: Schedule redeploy daemon mgr.x 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[28528]: from='client.24901 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[28528]: Schedule redeploy daemon mgr.x 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:14.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:57:14.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:57:14.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:13 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:13 smithi121 ceph-mon[24681]: from='client.24901 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:13 smithi121 ceph-mon[24681]: Schedule redeploy daemon mgr.x 2024-09-13T21:57:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:13 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:13 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:13 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:57:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:13 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:57:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:13 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:15.046 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:14 smithi114 ceph-mon[23730]: Deploying daemon mgr.x on smithi121 2024-09-13T21:57:15.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:14 smithi114 ceph-mon[23730]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:15.047 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:14 smithi114 ceph-mon[28528]: Deploying daemon mgr.x on smithi121 2024-09-13T21:57:15.047 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:14 smithi114 ceph-mon[28528]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:15.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:15] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-13T21:57:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:14 smithi121 ceph-mon[24681]: Deploying daemon mgr.x on smithi121 2024-09-13T21:57:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:14 smithi121 ceph-mon[24681]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:15.469 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:57:15.469 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 18s ago 9m - - 2024-09-13T21:57:15.469 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 10s ago 9m - - 2024-09-13T21:57:15.469 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (4m) 18s ago 4m 27.8M - ba2b418f427c 9bf04fad0c85 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (4m) 10s ago 4m 51.7M - 8.3.5 dad864ee21e9 b8bdbef3d489 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (3m) 18s ago 3m 52.8M - 3.5 e1d6a67b021e d4d9db680881 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443 running (9m) 10s ago 9m 412M - 17.2.0 e1d6a67b021e 7f14340b5cbc 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:9283 running (10m) 18s ago 10m 462M - 17.2.0 e1d6a67b021e f0c0d3e1c024 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (10m) 18s ago 10m 49.6M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (9m) 10s ago 9m 43.1M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (9m) 18s ago 9m 39.6M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (5m) 18s ago 5m 15.7M - 1dbe0e931976 45aa09d329e6 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (5m) 10s ago 5m 17.7M - 1dbe0e931976 5452c9d8e4cb 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (8m) 18s ago 8m 52.1M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (8m) 18s ago 8m 51.8M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (8m) 18s ago 8m 45.6M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (7m) 18s ago 7m 46.8M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T21:57:15.470 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (7m) 10s ago 7m 47.6M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T21:57:15.471 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (6m) 10s ago 6m 48.6M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T21:57:15.471 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (6m) 10s ago 6m 45.7M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T21:57:15.471 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (6m) 10s ago 6m 47.4M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T21:57:15.471 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (4m) 10s ago 5m 67.0M - 514e6a882f6e baadd3f398d4 2024-09-13T21:57:15.471 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (3m) 18s ago 3m 69.9M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T21:57:15.471 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (3m) 10s ago 3m 71.1M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T21:57:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:15 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:15 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:16.258 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T21:57:16.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:15 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:17.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:57:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:57:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[23730]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[23730]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[28528]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:17.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[28528]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:17.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:17.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:16 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:16 smithi121 ceph-mon[24681]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:16 smithi121 ceph-mon[24681]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:57:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:16 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:16 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:18.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:57:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:18 smithi114 ceph-mon[23730]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:18 smithi114 ceph-mon[28528]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:19.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:18 smithi121 ceph-mon[24681]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:20 smithi114 ceph-mon[23730]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:20 smithi114 ceph-mon[28528]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:20 smithi121 ceph-mon[24681]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:23.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:22 smithi114 ceph-mon[23730]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:23.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:22 smithi114 ceph-mon[28528]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:22 smithi121 ceph-mon[24681]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:24.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:57:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:57:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:24 smithi121 ceph-mon[24681]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:24 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:24 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:25.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:24 smithi114 ceph-mon[23730]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:25.236 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:24 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:25.236 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:24 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:25.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:24 smithi114 ceph-mon[28528]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:25.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:24 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:25.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:24 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:25.504 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:25] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-13T21:57:26.281 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:25 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:26.288 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:25 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:26.289 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:25 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:27.317 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:26 smithi121 ceph-mon[24681]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:27.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:26 smithi114 ceph-mon[23730]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:26 smithi114 ceph-mon[28528]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:28.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:57:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:28 smithi114 ceph-mon[23730]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:29.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:28 smithi114 ceph-mon[28528]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:28 smithi121 ceph-mon[24681]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:31.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:30 smithi114 ceph-mon[23730]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:30 smithi114 ceph-mon[28528]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:31.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:30 smithi121 ceph-mon[24681]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:33.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:32 smithi114 ceph-mon[23730]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:33.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:32 smithi114 ceph-mon[28528]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:32 smithi121 ceph-mon[24681]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:35.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:34 smithi114 ceph-mon[23730]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:35.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:34 smithi114 ceph-mon[28528]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:35.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:34 smithi121 ceph-mon[24681]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:35.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:35] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-09-13T21:57:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:35 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:57:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:57:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:35 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:57:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:57:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:35 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:57:36.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:57:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:36 smithi121 ceph-mon[24681]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:36 smithi114 ceph-mon[23730]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:37.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:57:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:57:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:36 smithi114 ceph-mon[28528]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:38.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[26209]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:57:39.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:38 smithi114 ceph-mon[23730]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:38 smithi114 ceph-mon[28528]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:38 smithi121 ceph-mon[24681]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:40 smithi114 ceph-mon[23730]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:40 smithi114 ceph-mon[28528]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:40 smithi121 ceph-mon[24681]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:43.256 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:42 smithi121 ceph-mon[24681]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:43.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:42 smithi114 ceph-mon[23730]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:43.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:42 smithi114 ceph-mon[28528]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:43.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:43 smithi121 systemd[1]: Stopping Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:57:43.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:43 smithi121 bash[49061]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr.x" found: no such container 2024-09-13T21:57:43.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:43 smithi121 podman[49072]: 2024-09-13 21:57:43.42641716 +0000 UTC m=+0.137929741 container died 7f14340b5cbc33b2393bc01600a2adbb3ce4720d1386ef59b0b93c39e70a1dd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, version=8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-13T21:57:43.863 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:43 smithi121 podman[49072]: 2024-09-13 21:57:43.839549893 +0000 UTC m=+0.551062462 container cleanup 7f14340b5cbc33b2393bc01600a2adbb3ce4720d1386ef59b0b93c39e70a1dd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., version=8, io.buildah.version=1.19.8, architecture=x86_64, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, release=754, io.openshift.expose-services=, vcs-type=git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T21:57:43.863 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:43 smithi121 bash[49072]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x 2024-09-13T21:57:43.863 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:43 smithi121 bash[49100]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr.x" found: no such container 2024-09-13T21:57:44.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 podman[49084]: 2024-09-13 21:57:44.03516605 +0000 UTC m=+0.603978217 container remove 7f14340b5cbc33b2393bc01600a2adbb3ce4720d1386ef59b0b93c39e70a1dd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , name=centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, GIT_CLEAN=True, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T21:57:44.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T21:57:44.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T21:57:44.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 systemd[1]: Stopped Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:57:44.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Consumed 17.300s CPU time. 2024-09-13T21:57:44.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 systemd[1]: Starting Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T21:57:44.706 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 podman[49200]: 2024-09-13 21:57:44.350716127 +0000 UTC m=+0.015458949 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:57:44.706 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 podman[49200]: 2024-09-13 21:57:44.458707076 +0000 UTC m=+0.123449930 container create 56081b4c77470436765fc94d3dfe84d5aba0c513957d76556ae8d37988eb2b19 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T21:57:44.706 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 podman[49200]: 2024-09-13 21:57:44.677883659 +0000 UTC m=+0.342626478 container init 56081b4c77470436765fc94d3dfe84d5aba0c513957d76556ae8d37988eb2b19 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-13T21:57:44.706 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 podman[49200]: 2024-09-13 21:57:44.680708226 +0000 UTC m=+0.345451039 container start 56081b4c77470436765fc94d3dfe84d5aba0c513957d76556ae8d37988eb2b19 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:57:44.999 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mgr[49218]: -- 172.21.15.121:0/576369964 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a167da30e0 con 0x55a167d80800 2024-09-13T21:57:44.999 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 bash[49200]: 56081b4c77470436765fc94d3dfe84d5aba0c513957d76556ae8d37988eb2b19 2024-09-13T21:57:44.999 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 systemd[1]: Started Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T21:57:44.999 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:44.834+0000 7f338fcab040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T21:57:44.999 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:44.901+0000 7f338fcab040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T21:57:45.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:57:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:57:45.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-mon[23730]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:44 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-mon[28528]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:45.291 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:45.060+0000 7f338fcab040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T21:57:45.291 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:45.129+0000 7f338fcab040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T21:57:45.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mon[24681]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:45.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:45.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:57:45.292 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:44 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:57:45.407 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:45] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-09-13T21:57:45.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:45.297+0000 7f338fcab040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:57:45.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:45.441+0000 7f338fcab040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T21:57:46.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:46 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:46.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:46.033 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:46.033 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:45.772+0000 7f338fcab040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T21:57:46.033 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:45.835+0000 7f338fcab040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T21:57:46.033 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:46.031+0000 7f338fcab040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T21:57:46.170 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:46 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:46.170 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:46.170 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:46.171 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:46 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:46.171 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:46.171 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:47.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:47 smithi121 ceph-mon[24681]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:47.294 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:47.004+0000 7f338fcab040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T21:57:47.294 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:47.078+0000 7f338fcab040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T21:57:47.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:47 smithi114 ceph-mon[23730]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:47.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:47 smithi114 ceph-mon[28528]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:48.078 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:47.814+0000 7f338fcab040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T21:57:48.078 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:47.931+0000 7f338fcab040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T21:57:48.358 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.076+0000 7f338fcab040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T21:57:48.358 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.138+0000 7f338fcab040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T21:57:48.358 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.288+0000 7f338fcab040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T21:57:48.358 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.356+0000 7f338fcab040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T21:57:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.417+0000 7f338fcab040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T21:57:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.575+0000 7f338fcab040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T21:57:48.910 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.640+0000 7f338fcab040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T21:57:48.910 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.708+0000 7f338fcab040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T21:57:48.910 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.790+0000 7f338fcab040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T21:57:48.910 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.849+0000 7f338fcab040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T21:57:48.910 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:48.908+0000 7f338fcab040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T21:57:49.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:49 smithi114 ceph-mon[23730]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:49.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:49 smithi114 ceph-mon[28528]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-mon[24681]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:50.012 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:49.722+0000 7f338fcab040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T21:57:50.012 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:21:57:49] ENGINE Bus STARTING 2024-09-13T21:57:50.012 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: CherryPy Checker: 2024-09-13T21:57:50.012 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: The Application mounted at '' has an empty config. 2024-09-13T21:57:50.012 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T21:57:50.012 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:21:57:49] ENGINE Serving on http://:::9283 2024-09-13T21:57:50.013 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:49 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:21:57:49] ENGINE Bus STARTED 2024-09-13T21:57:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[23730]: Standby manager daemon x restarted 2024-09-13T21:57:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T21:57:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:57:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:57:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:57:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:57:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[28528]: Standby manager daemon x restarted 2024-09-13T21:57:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T21:57:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:57:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:57:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:57:50.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:50 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:57:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:50 smithi121 ceph-mon[24681]: Standby manager daemon x restarted 2024-09-13T21:57:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:50 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T21:57:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:50 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T21:57:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:50 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:57:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:50 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T21:57:50.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:50 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:57:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:51 smithi114 ceph-mon[23730]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:51 smithi114 ceph-mon[23730]: mgrmap e21: y(active, since 6m), standbys: x 2024-09-13T21:57:51.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:51 smithi114 ceph-mon[28528]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:51.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:51 smithi114 ceph-mon[28528]: mgrmap e21: y(active, since 6m), standbys: x 2024-09-13T21:57:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:51 smithi121 ceph-mon[24681]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:51 smithi121 ceph-mon[24681]: mgrmap e21: y(active, since 6m), standbys: x 2024-09-13T21:57:53.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:53 smithi114 ceph-mon[23730]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:53 smithi114 ceph-mon[28528]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:53 smithi121 ceph-mon[24681]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:55.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:55 smithi114 ceph-mon[23730]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:55.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:55 smithi114 ceph-mon[28528]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:55 smithi121 ceph-mon[24681]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:55.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:55] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-09-13T21:57:56.282 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:56 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:56.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:56 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:56.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:56 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:57:57.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:57 smithi114 ceph-mon[23730]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:57.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:57 smithi114 ceph-mon[28528]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:57.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:57:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:57:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:57:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:57 smithi121 ceph-mon[24681]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:57:58.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:58.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:58.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:58.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:58.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:58.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:57:58.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:57:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:57:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:57:59.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:57:59 smithi114 ceph-mon[23730]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:59.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:57:59 smithi114 ceph-mon[28528]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:57:59.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:57:59 smithi121 ceph-mon[24681]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:01.115 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:01 smithi121 ceph-mon[24681]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:01.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:01 smithi114 ceph-mon[23730]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:01 smithi114 ceph-mon[28528]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:03.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:03 smithi114 ceph-mon[23730]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:03.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:03 smithi114 ceph-mon[28528]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:03.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:03 smithi121 ceph-mon[24681]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:03.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:03.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:05.105 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:05 smithi114 ceph-mon[23730]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:05.106 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:05.106 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:05.106 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:58:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:58:05.106 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:05 smithi114 ceph-mon[28528]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:05.106 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:05.106 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:05.308 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:05 smithi121 ceph-mon[24681]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:05.309 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:05.309 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:05.421 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:05] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-13T21:58:06.154 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:06 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:06.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:06 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:06.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:06 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:06.583 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:58:06] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:58:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:07 smithi121 ceph-mon[24681]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:07 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:07 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:07 smithi114 ceph-mon[23730]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:07 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:07.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:07 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:07.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:07 smithi114 ceph-mon[28528]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:07.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:07 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:07.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:07 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:08.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:58:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:58:09.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:09 smithi121 ceph-mon[24681]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:09.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:09 smithi114 ceph-mon[23730]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:09.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:09 smithi114 ceph-mon[28528]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:11.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:11 smithi121 ceph-mon[24681]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:11.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:11 smithi114 ceph-mon[23730]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:11.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:11 smithi114 ceph-mon[28528]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:13 smithi121 ceph-mon[24681]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:13.507 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:13 smithi114 ceph-mon[23730]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:13.508 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:13 smithi114 ceph-mon[28528]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:13.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:58:13.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:13.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:13.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:13.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:13.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:15 smithi121 ceph-mon[24681]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:15 smithi114 ceph-mon[23730]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:15.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:15 smithi114 ceph-mon[28528]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:15.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:15] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-13T21:58:16.327 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:16 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:16.475 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:16 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:16.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:16 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:17.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:17 smithi121 ceph-mon[24681]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:17.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:17 smithi114 ceph-mon[23730]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:17.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:17 smithi114 ceph-mon[28528]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:17.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:58:17] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:58:18.437 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:58:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:58:18.437 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:18.437 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:18.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:18 smithi121 ceph-mon[24681]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:18.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:18 smithi114 ceph-mon[23730]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:18.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:18 smithi114 ceph-mon[28528]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:20.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:20 smithi114 ceph-mon[23730]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:20.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:20 smithi114 ceph-mon[28528]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:20 smithi121 ceph-mon[24681]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:22.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:22 smithi121 ceph-mon[24681]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:23.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:22 smithi114 ceph-mon[23730]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:22 smithi114 ceph-mon[28528]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:23.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:58:23.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:23.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:23.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:23.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:24.843 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:24 smithi114 ceph-mon[23730]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:24.844 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:24 smithi114 ceph-mon[28528]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:24.844 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:58:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:58:24.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:24 smithi121 ceph-mon[24681]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:25.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:25] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-09-13T21:58:25.917 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:25 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:25.917 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:25 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:25.917 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:25 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:25.917 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:25 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:25.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:25 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:25.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:25 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:25 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:25 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:25 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:26.978 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:26 smithi114 ceph-mon[23730]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:26.979 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:26 smithi114 ceph-mon[28528]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:27.023 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:26 smithi121 ceph-mon[24681]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:28.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:58:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:58:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:28 smithi114 ceph-mon[23730]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:28 smithi114 ceph-mon[28528]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:29.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:28 smithi121 ceph-mon[24681]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:30 smithi114 ceph-mon[23730]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:31.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:30 smithi114 ceph-mon[28528]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:30 smithi121 ceph-mon[24681]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:32 smithi121 ceph-mon[24681]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:33.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:32 smithi114 ceph-mon[23730]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:33.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:32 smithi114 ceph-mon[28528]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:33.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:58:33.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:33.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:33.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:33.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:34.817 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:34 smithi121 ceph-mon[24681]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:35.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:34 smithi114 ceph-mon[23730]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:35.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:34 smithi114 ceph-mon[28528]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:35.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:35] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-09-13T21:58:36.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:35 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:58:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:35 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:58:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:35 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:58:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:35 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:58:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:35 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:58:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:35 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:58:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:36 smithi114 ceph-mon[23730]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:37.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:36 smithi114 ceph-mon[28528]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:36 smithi121 ceph-mon[24681]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:37.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:58:37] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:58:38.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:38 smithi114 ceph-mon[23730]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:38.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:38 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:38.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:38 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:38.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:38 smithi114 ceph-mon[28528]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:38.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:38 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:38.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:38 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:38.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:58:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:58:38.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:38 smithi121 ceph-mon[24681]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:38.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:38 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:38.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:38 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:40.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:40 smithi114 ceph-mon[23730]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:40.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:40 smithi114 ceph-mon[28528]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:40 smithi121 ceph-mon[24681]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:42.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:42 smithi121 ceph-mon[24681]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:42 smithi114 ceph-mon[23730]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:43.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:42 smithi114 ceph-mon[28528]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:43.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:58:43.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:43.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:43.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:44.858 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:44 smithi114 ceph-mon[23730]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:44.858 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:44 smithi114 ceph-mon[28528]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:44.859 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:58:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:58:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:44 smithi121 ceph-mon[24681]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:45.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:45] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-09-13T21:58:45.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:45.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:45.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:45.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:58:45.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:58:45.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:45.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:45.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:45.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:45.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:58:45.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:58:45.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:45 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:45 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:45 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:45 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:45 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:58:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:45 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:58:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:45 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:46 smithi114 ceph-mon[23730]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:47.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:46 smithi114 ceph-mon[28528]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:46 smithi121 ceph-mon[24681]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:58:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:58:49.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:48 smithi114 ceph-mon[23730]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:48 smithi114 ceph-mon[28528]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:48 smithi121 ceph-mon[24681]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:50 smithi114 ceph-mon[23730]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:50 smithi114 ceph-mon[28528]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:50 smithi121 ceph-mon[24681]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:52 smithi114 ceph-mon[23730]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:52 smithi114 ceph-mon[28528]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:52 smithi121 ceph-mon[24681]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:53.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:58:53.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:53.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:58:53.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:58:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:58:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:58:54.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:54 smithi121 ceph-mon[24681]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:55.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:54 smithi114 ceph-mon[23730]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:55.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:54 smithi114 ceph-mon[28528]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:55.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:55] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-13T21:58:56.009 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:55 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:56.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:55 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:56.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:55 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:58:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:56 smithi121 ceph-mon[24681]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:57.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:56 smithi114 ceph-mon[23730]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:57.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:56 smithi114 ceph-mon[28528]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:58:57.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:58:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:58:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:58 smithi121 ceph-mon[24681]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:58:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:58.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:58:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:58:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:58:58.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:58 smithi114 ceph-mon[23730]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:58.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:58.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:58:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:58 smithi114 ceph-mon[28528]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:58:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:58:58.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:58:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:00.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:00 smithi121 ceph-mon[24681]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:00.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:00 smithi114 ceph-mon[23730]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:00.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:00 smithi114 ceph-mon[28528]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:02.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:02 smithi121 ceph-mon[24681]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:03.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:02 smithi114 ceph-mon[23730]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:03.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:02 smithi114 ceph-mon[28528]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:03.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:59:03.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:03.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:03.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:04.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:04 smithi121 ceph-mon[24681]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:04.875 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:04 smithi114 ceph-mon[23730]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:04.876 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:04 smithi114 ceph-mon[28528]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:05.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:59:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:59:05.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:05] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-09-13T21:59:06.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:05 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:06.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:06.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:06.157 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:05 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:06.157 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:06.158 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:06.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:05 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:06.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:06.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:07.196 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:06 smithi114 ceph-mon[23730]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:07.197 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:06 smithi114 ceph-mon[28528]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:06 smithi121 ceph-mon[24681]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:08.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:59:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:59:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:08 smithi114 ceph-mon[23730]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:09.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:08 smithi114 ceph-mon[28528]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:09.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:08 smithi121 ceph-mon[24681]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:10 smithi114 ceph-mon[23730]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:10 smithi114 ceph-mon[28528]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:11.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:10 smithi121 ceph-mon[24681]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:12 smithi114 ceph-mon[23730]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:12 smithi114 ceph-mon[28528]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:12 smithi121 ceph-mon[24681]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:13.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:59:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:13.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:13.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:15.234 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:14 smithi121 ceph-mon[24681]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:15.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:14 smithi114 ceph-mon[23730]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:15.235 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:14 smithi114 ceph-mon[28528]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:15.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:15] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-09-13T21:59:16.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:15 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:16.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:15 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:16.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:15 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:17.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:16 smithi114 ceph-mon[23730]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:16 smithi114 ceph-mon[28528]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:16 smithi121 ceph-mon[24681]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:17.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:59:17] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:59:18.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:59:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:59:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:18 smithi121 ceph-mon[24681]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:18.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:18 smithi114 ceph-mon[28528]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:18.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:18.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:18 smithi114 ceph-mon[23730]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:18.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:18.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:20.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:20 smithi114 ceph-mon[23730]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:20.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:20 smithi114 ceph-mon[28528]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:20.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:20 smithi121 ceph-mon[24681]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:22.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:22 smithi121 ceph-mon[24681]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:23.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:22 smithi114 ceph-mon[23730]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:22 smithi114 ceph-mon[28528]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:23.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:59:23.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:23.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:23.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:23.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:23.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:24.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:24 smithi121 ceph-mon[24681]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:24.891 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:24 smithi114 ceph-mon[23730]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:24.891 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:24 smithi114 ceph-mon[28528]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:25.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:59:25] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:59:25.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:25] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-09-13T21:59:26.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:26 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:26.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:26 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:26.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:26 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:26 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:26 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:26.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:26 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:26 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:26 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:26 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:27.162 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:27 smithi114 ceph-mon[23730]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:27.163 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:27 smithi114 ceph-mon[28528]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:27.273 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:27 smithi121 ceph-mon[24681]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:28.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:59:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:59:29.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:29 smithi114 ceph-mon[28528]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:29 smithi114 ceph-mon[23730]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:29 smithi121 ceph-mon[24681]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:31.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:31 smithi114 ceph-mon[23730]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:31 smithi114 ceph-mon[28528]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:31.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:31 smithi121 ceph-mon[24681]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:33.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:33 smithi114 ceph-mon[23730]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:33.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:33 smithi114 ceph-mon[28528]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:33 smithi121 ceph-mon[24681]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:33.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:59:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:33.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:33.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:35.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:35 smithi121 ceph-mon[24681]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:35.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:35 smithi114 ceph-mon[23730]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:35.235 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:35 smithi114 ceph-mon[28528]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:35.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:35] "GET /metrics HTTP/1.1" 200 207985 "" "Prometheus/2.33.4" 2024-09-13T21:59:36.210 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:36 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:36.210 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:59:36.210 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:36 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:59:36.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:36 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:36.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:59:36.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:36 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:59:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:36 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:59:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:36 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T21:59:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:37 smithi114 ceph-mon[23730]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:37 smithi114 ceph-mon[28528]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:37 smithi121 ceph-mon[24681]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:37.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:59:37] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:59:38.556 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:59:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:59:38.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:38 smithi114 ceph-mon[23730]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:38.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:38 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:38.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:38 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:38.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:38 smithi114 ceph-mon[28528]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:38.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:38 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:38.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:38 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:38.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:38 smithi121 ceph-mon[24681]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:38.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:38 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:38.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:38 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:40.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:40 smithi114 ceph-mon[23730]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:40.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:40 smithi114 ceph-mon[28528]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:40.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:40 smithi121 ceph-mon[24681]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:42.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:42 smithi121 ceph-mon[24681]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:42 smithi114 ceph-mon[23730]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:43.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:42 smithi114 ceph-mon[28528]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:43.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:59:43.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:43.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:43.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:44 smithi121 ceph-mon[24681]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:44.912 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:44 smithi114 ceph-mon[23730]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:44.913 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:44 smithi114 ceph-mon[28528]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:45.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:21:59:45] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:59:45.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:45] "GET /metrics HTTP/1.1" 200 207985 "" "Prometheus/2.33.4" 2024-09-13T21:59:46.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:46 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:46.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:59:46.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:59:46.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:46 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:46.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.477 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:46.477 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:59:46.477 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:59:46.477 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:46 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:47 smithi121 ceph-mon[24681]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:47.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:47 smithi114 ceph-mon[23730]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:47.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:47 smithi114 ceph-mon[28528]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:59:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:59:49.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:49 smithi114 ceph-mon[23730]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:49.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:49 smithi114 ceph-mon[28528]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:49.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:49 smithi121 ceph-mon[24681]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:50.528 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:50 smithi114 ceph-mon[23730]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:50.528 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:50 smithi114 ceph-mon[28528]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:50.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:50 smithi121 ceph-mon[24681]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:52.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:52 smithi121 ceph-mon[24681]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:52 smithi114 ceph-mon[23730]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:53.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:52 smithi114 ceph-mon[28528]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:53.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T21:59:53.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:53.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T21:59:53.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 21:59:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T21:59:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T21:59:54.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:54 smithi121 ceph-mon[24681]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:54.920 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:54 smithi114 ceph-mon[23730]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:54.920 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:54 smithi114 ceph-mon[28528]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:55.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:55] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-09-13T21:59:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:55 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:56.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:55 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:56.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:55 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T21:59:56.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:56 smithi121 ceph-mon[24681]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:56 smithi114 ceph-mon[23730]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:56 smithi114 ceph-mon[28528]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T21:59:58.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 21:59:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:21:59:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T21:59:58.425 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 21:59:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:21:59:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T21:59:58.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:58 smithi121 ceph-mon[24681]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:58.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:58.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 21:59:58 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:58 smithi114 ceph-mon[23730]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 21:59:58 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:58 smithi114 ceph-mon[28528]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T21:59:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T21:59:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 21:59:58 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:01.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:00 smithi114 ceph-mon[23730]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:01.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:00 smithi114 ceph-mon[23730]: overall HEALTH_OK 2024-09-13T22:00:01.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:00 smithi114 ceph-mon[28528]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:01.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:00 smithi114 ceph-mon[28528]: overall HEALTH_OK 2024-09-13T22:00:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:00 smithi121 ceph-mon[24681]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:01.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:00 smithi121 ceph-mon[24681]: overall HEALTH_OK 2024-09-13T22:00:02.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:02 smithi121 ceph-mon[24681]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:03.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:02 smithi114 ceph-mon[23730]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:03.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:02 smithi114 ceph-mon[28528]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:03.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T22:00:03.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:03.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:03.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:04.930 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:04 smithi114 ceph-mon[23730]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:04.930 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:04 smithi114 ceph-mon[28528]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:05.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:04 smithi121 ceph-mon[24681]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:05.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:22:00:05] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T22:00:05.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:05] "GET /metrics HTTP/1.1" 200 207989 "" "Prometheus/2.33.4" 2024-09-13T22:00:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:05 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:05 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:05 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:06 smithi114 ceph-mon[23730]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:06 smithi114 ceph-mon[28528]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:06 smithi121 ceph-mon[24681]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:00:08.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T22:00:08.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:08 smithi114 ceph-mon[23730]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:08.964 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:08 smithi114 ceph-mon[28528]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:08 smithi121 ceph-mon[24681]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:10 smithi114 ceph-mon[23730]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:10 smithi114 ceph-mon[28528]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:10 smithi121 ceph-mon[24681]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:12 smithi114 ceph-mon[23730]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:13.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:12 smithi114 ceph-mon[28528]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:12 smithi121 ceph-mon[24681]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:13.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T22:00:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:13.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:13.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:14.940 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:14 smithi114 ceph-mon[23730]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:14.941 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:14 smithi114 ceph-mon[28528]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:14 smithi121 ceph-mon[24681]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:15.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:15 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:15] "GET /metrics HTTP/1.1" 200 207989 "" "Prometheus/2.33.4" 2024-09-13T22:00:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:15 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:15 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:15 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:16 smithi114 ceph-mon[23730]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:16 smithi114 ceph-mon[28528]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:17.077 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:16 smithi121 ceph-mon[24681]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:17.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.114 - - [13/Sep/2024:22:00:17] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T22:00:18.441 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T22:00:18.587 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:00:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:18 smithi114 ceph-mon[23730]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:18 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:18 smithi114 ceph-mon[28528]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:18 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:18 smithi121 ceph-mon[24681]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:18 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 2s ago 12m - - 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 15s ago 12m - - 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (7m) 2s ago 7m 27.2M - ba2b418f427c 9bf04fad0c85 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (7m) 15s ago 7m 52.3M - 8.3.5 dad864ee21e9 b8bdbef3d489 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (6m) 2s ago 6m 60.4M - 3.5 e1d6a67b021e d4d9db680881 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283 running (2m) 15s ago 12m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 56081b4c7747 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:9283 running (13m) 2s ago 13m 470M - 17.2.0 e1d6a67b021e f0c0d3e1c024 2024-09-13T22:00:20.594 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (13m) 2s ago 14m 54.6M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (12m) 15s ago 12m 45.9M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (12m) 2s ago 12m 43.0M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (8m) 2s ago 8m 18.6M - 1dbe0e931976 45aa09d329e6 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (8m) 15s ago 8m 21.2M - 1dbe0e931976 5452c9d8e4cb 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (11m) 2s ago 11m 53.8M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (11m) 2s ago 11m 53.4M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (11m) 2s ago 11m 47.0M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (10m) 2s ago 10m 48.5M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (10m) 15s ago 10m 49.4M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (9m) 15s ago 9m 50.3M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (9m) 15s ago 9m 47.1M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (9m) 15s ago 9m 48.8M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (7m) 15s ago 8m 67.2M - 514e6a882f6e baadd3f398d4 2024-09-13T22:00:20.595 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (6m) 2s ago 6m 70.8M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:00:20.596 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (6m) 15s ago 6m 72.0M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:00:20.897 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:20 smithi114 ceph-mon[23730]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:20.897 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:20 smithi114 ceph-mon[28528]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:20 smithi121 ceph-mon[24681]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:21.205 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T22:00:22.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:21 smithi114 ceph-mon[23730]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:00:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:21 smithi114 ceph-mon[28528]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:00:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:21 smithi121 ceph-mon[24681]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:00:23.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:22 smithi114 ceph-mon[23730]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:22 smithi114 ceph-mon[28528]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:22 smithi121 ceph-mon[24681]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:23.129 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:00:23.129 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:00:23.129 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:00:23.129 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-09-13T22:00:23.130 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:00:23.131 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:00:23.131 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:00:23.770 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T22:00:23.793 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:23 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/33666315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:00:23.793 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:23 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/33666315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:00:23.793 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T22:00:23.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:23.793 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:23.794 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:24.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:23 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/33666315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:00:25.042 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:24 smithi114 ceph-mon[28528]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:25.043 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:24 smithi114 ceph-mon[23730]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:24 smithi121 ceph-mon[24681]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:25.313 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:25] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-13T22:00:25.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 172.21.15.121 - - [13/Sep/2024:22:00:25] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-09-13T22:00:25.767 INFO:teuthology.orchestra.run.smithi114.stdout: cluster: 2024-09-13T22:00:25.767 INFO:teuthology.orchestra.run.smithi114.stdout: id: 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:00:25.767 INFO:teuthology.orchestra.run.smithi114.stdout: health: HEALTH_OK 2024-09-13T22:00:25.767 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:00:25.767 INFO:teuthology.orchestra.run.smithi114.stdout: services: 2024-09-13T22:00:25.767 INFO:teuthology.orchestra.run.smithi114.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-09-13T22:00:25.767 INFO:teuthology.orchestra.run.smithi114.stdout: mgr: y(active, since 8m), standbys: x 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: osd: 8 osds: 8 up (since 9m), 8 in (since 9m) 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: data: 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: pools: 6 pools, 161 pgs 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: objects: 209 objects, 585 KiB 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: pgs: 161 active+clean 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: io: 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-13T22:00:25.768 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:00:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-mon[23730]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:26.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:26.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:26.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:25 smithi114 ceph-mon[28528]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:25 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:25 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:25 smithi121 ceph-mon[24681]: from='mgr.24346 172.21.15.114:0/3082681911' entity='mgr.y' 2024-09-13T22:00:26.439 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T22:00:27.035 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:26 smithi114 ceph-mon[28528]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:27.035 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:26 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2957404196' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:00:27.036 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:26 smithi114 ceph-mon[23730]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:27.036 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:26 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2957404196' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:00:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:26 smithi121 ceph-mon[24681]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:26 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2957404196' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:00:28.426 INFO:teuthology.orchestra.run.smithi114.stdout:HEALTH_OK 2024-09-13T22:00:28.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T22:00:29.003 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-09-13T22:00:29.027 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:28 smithi114 ceph-mon[23730]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:29.027 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:28 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/565531928' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:00:29.028 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:28 smithi114 ceph-mon[28528]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:29.028 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:28 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/565531928' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:00:29.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:28 smithi121 ceph-mon[24681]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:29.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:28 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/565531928' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:00:30.864 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:00:30.992 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:30 smithi114 ceph-mon[23730]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:30.992 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:30 smithi114 ceph-mon[28528]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:30 smithi121 ceph-mon[24681]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:31.806 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr fail' 2024-09-13T22:00:32.041 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:31 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3613945259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:00:32.041 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:31 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3613945259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:00:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:31 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3613945259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:00:32.929 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:32 smithi114 ceph-mon[23730]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:32.930 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:32 smithi114 ceph-mon[28528]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:33.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:32 smithi121 ceph-mon[24681]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:33.734 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T22:00:33.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:33.735 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:33.735 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:35.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[23730]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:35.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/421995480' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:00:35.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:00:35.048 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[23730]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T22:00:35.048 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setuser ceph since I am not root 2024-09-13T22:00:35.048 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ignoring --setgroup ceph since I am not root 2024-09-13T22:00:35.048 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mgr[23963]: -- 172.21.15.114:0/1764203000 <== mon.1 v2:172.21.15.114:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55aae463a340 con 0x55aae4613000 2024-09-13T22:00:35.048 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:34.965+0000 7f5ccbe18000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:00:35.048 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[28528]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:35.048 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/421995480' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:00:35.049 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:00:35.049 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:34 smithi114 ceph-mon[28528]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T22:00:35.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:00:34] ENGINE Bus STOPPING 2024-09-13T22:00:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:34 smithi121 ceph-mon[24681]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:34 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/421995480' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:00:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:34 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:00:35.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:34 smithi121 ceph-mon[24681]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T22:00:35.315 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:35.238Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:00:35.316 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:35.045+0000 7f5ccbe18000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:00:35.316 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:35.222+0000 7f5ccbe18000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:00:35.316 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:35.314+0000 7f5ccbe18000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:00:35.392 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T22:00:35.570 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:35.470+0000 7f5ccbe18000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:00:35.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:00:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:00:35.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:00:35] ENGINE Bus STOPPED 2024-09-13T22:00:35.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:00:35] ENGINE Bus STARTING 2024-09-13T22:00:35.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:00:35] ENGINE Serving on http://:::9283 2024-09-13T22:00:35.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:00:35] ENGINE Bus STARTED 2024-09-13T22:00:35.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:35.726+0000 7f5ccbe18000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: mgrmap e22: x(active, starting, since 0.194235s) 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:00:35.822 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: Manager daemon x is now available 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: Queued rgw.foo for migration 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.823 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: Checking for cert/key for grafana.a 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: mgrmap e22: x(active, starting, since 0.194235s) 2024-09-13T22:00:35.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:00:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: Manager daemon x is now available 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: Queued rgw.foo for migration 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: Checking for cert/key for grafana.a 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:00:35.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:00:35.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:00:35.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:00:35.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:00:35.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:00:35.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:00:35.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:00:35.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:00:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: mgrmap e22: x(active, starting, since 0.194235s) 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:00:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: Manager daemon x is now available 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: Queued rgw.foo for migration 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: Checking for cert/key for grafana.a 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:00:36.361 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:36.090+0000 7f5ccbe18000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:00:36.361 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:36.185+0000 7f5ccbe18000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:00:36.361 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:36.359+0000 7f5ccbe18000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:00:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: mgrmap e23: x(active, since 1.20949s) 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: mgrmap e23: x(active, since 1.20949s) 2024-09-13T22:00:37.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:00:37.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:00:37.108 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:00:36.990+0000 7f32e6c5a640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:22:00:36] ENGINE Error in HTTPServer.serve 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self._connections.run(self.expiration_interval) 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self._run(expiration_interval) 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: s = self.context.wrap_socket( 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:00:37.109 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.sslsocket_class._create( 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self.do_handshake() 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self._sslobj.do_handshake() 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: mgrmap e23: x(active, since 1.20949s) 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:00:37.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:37.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:00:37.112 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:00:37.547 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:37.454+0000 7f5ccbe18000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:00:38.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:37.547+0000 7f5ccbe18000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:00:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: [13/Sep/2024:22:00:36] ENGINE Bus STARTING 2024-09-13T22:00:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: [13/Sep/2024:22:00:36] ENGINE Serving on http://172.21.15.121:8765 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: [13/Sep/2024:22:00:36] ENGINE Serving on https://172.21.15.121:7150 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: [13/Sep/2024:22:00:36] ENGINE Bus STARTED 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: [13/Sep/2024:22:00:36] ENGINE Error in HTTPServer.serve 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:00:38.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.do_handshake() 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: Reconfiguring daemon agent.smithi114 on smithi114 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[23730]: Deploying cephadm binary to smithi114 2024-09-13T22:00:38.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: [13/Sep/2024:22:00:36] ENGINE Bus STARTING 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: [13/Sep/2024:22:00:36] ENGINE Serving on http://172.21.15.121:8765 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: [13/Sep/2024:22:00:36] ENGINE Serving on https://172.21.15.121:7150 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: [13/Sep/2024:22:00:36] ENGINE Bus STARTED 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: [13/Sep/2024:22:00:36] ENGINE Error in HTTPServer.serve 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:00:38.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.do_handshake() 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: Reconfiguring daemon agent.smithi114 on smithi114 2024-09-13T22:00:38.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:37 smithi114 ceph-mon[28528]: Deploying cephadm binary to smithi114 2024-09-13T22:00:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: [13/Sep/2024:22:00:36] ENGINE Bus STARTING 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: [13/Sep/2024:22:00:36] ENGINE Serving on http://172.21.15.121:8765 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: [13/Sep/2024:22:00:36] ENGINE Serving on https://172.21.15.121:7150 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: [13/Sep/2024:22:00:36] ENGINE Bus STARTED 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: [13/Sep/2024:22:00:36] ENGINE Error in HTTPServer.serve 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._run(expiration_interval) 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:00:38.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: s = self.context.wrap_socket( 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.sslsocket_class._create( 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.do_handshake() 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._sslobj.do_handshake() 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: Reconfiguring daemon agent.smithi114 on smithi114 2024-09-13T22:00:38.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:37 smithi121 ceph-mon[24681]: Deploying cephadm binary to smithi114 2024-09-13T22:00:38.471 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:38] "GET /metrics HTTP/1.1" 200 34910 "" "Prometheus/2.33.4" 2024-09-13T22:00:38.726 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:38.471+0000 7f5ccbe18000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:00:39.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[23730]: mgrmap e24: x(active, since 3s) 2024-09-13T22:00:39.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.009 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[28528]: mgrmap e24: x(active, since 3s) 2024-09-13T22:00:39.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.009 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:38.807+0000 7f5ccbe18000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:00:39.009 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:38.971+0000 7f5ccbe18000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:00:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:38 smithi121 ceph-mon[24681]: mgrmap e24: x(active, since 3s) 2024-09-13T22:00:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:38 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:38 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:38 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:39.283 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:39.088+0000 7f5ccbe18000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:00:39.283 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:39.188+0000 7f5ccbe18000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:00:39.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:39.282+0000 7f5ccbe18000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:00:39.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:39.437+0000 7f5ccbe18000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:00:39.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:39.532+0000 7f5ccbe18000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:00:39.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:39.633+0000 7f5ccbe18000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:00:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:39 smithi121 ceph-mon[24681]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T22:00:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:39 smithi121 ceph-mon[24681]: Deploying daemon node-exporter.a on smithi114 2024-09-13T22:00:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:39 smithi121 ceph-mon[24681]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:40.135 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:40 smithi114 systemd[1]: Stopping Ceph node-exporter.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:00:40.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-mon[23730]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T22:00:40.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-mon[23730]: Deploying daemon node-exporter.a on smithi114 2024-09-13T22:00:40.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-mon[23730]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:40.135 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-mon[28528]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T22:00:40.136 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-mon[28528]: Deploying daemon node-exporter.a on smithi114 2024-09-13T22:00:40.136 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-mon[28528]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:40.136 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:39.939+0000 7f5ccbe18000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:00:40.136 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:40.035+0000 7f5ccbe18000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:00:40.136 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:40.134+0000 7f5ccbe18000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:00:41.574 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:41 smithi114 podman[63174]: 2024-09-13 22:00:41.456765043 +0000 UTC m=+0.459098452 container cleanup 45aa09d329e6e03fdf9727f6a424e56446fcfe68f5c614f6c02946ea1bb0642e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:41.574 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:41 smithi114 bash[63174]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T22:00:41.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:41.267+0000 7f5ccbe18000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:00:41.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:22:00:41] ENGINE Bus STARTING 2024-09-13T22:00:41.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: CherryPy Checker: 2024-09-13T22:00:41.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: The Application mounted at '' has an empty config. 2024-09-13T22:00:41.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: 2024-09-13T22:00:41.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:22:00:41] ENGINE Serving on http://:::9283 2024-09-13T22:00:41.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: [13/Sep/2024:22:00:41] ENGINE Bus STARTED 2024-09-13T22:00:41.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:41 smithi121 ceph-mon[24681]: Standby manager daemon y started 2024-09-13T22:00:41.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:41 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:00:41.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:41 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:00:41.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:41 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:00:41.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:41 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:00:41.933 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:41 smithi114 podman[63185]: 2024-09-13 22:00:41.775179669 +0000 UTC m=+0.660558936 container remove 45aa09d329e6e03fdf9727f6a424e56446fcfe68f5c614f6c02946ea1bb0642e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:41.933 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:41 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[23730]: Standby manager daemon y started 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[28528]: Standby manager daemon y started 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:00:41.934 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:00:41.935 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:41 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/2025521514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:00:42.325 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:41 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T22:00:42.325 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:41 smithi114 systemd[1]: Stopped Ceph node-exporter.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:00:42.325 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:41 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 4.222s CPU time. 2024-09-13T22:00:42.325 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:42 smithi114 systemd[1]: Starting Ceph node-exporter.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:00:42.761 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:42 smithi114 bash[63464]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T22:00:42.761 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:42 smithi114 ceph-mon[23730]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:42.761 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:42 smithi114 ceph-mon[23730]: mgrmap e25: x(active, since 7s), standbys: y 2024-09-13T22:00:42.761 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:42 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:00:42.762 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:42 smithi114 ceph-mon[28528]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:42.762 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:42 smithi114 ceph-mon[28528]: mgrmap e25: x(active, since 7s), standbys: y 2024-09-13T22:00:42.762 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:42 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:00:42.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:42 smithi121 ceph-mon[24681]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:00:42.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:42 smithi121 ceph-mon[24681]: mgrmap e25: x(active, since 7s), standbys: y 2024-09-13T22:00:42.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:42 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:00:43.063 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:42 smithi114 bash[63464]: Getting image source signatures 2024-09-13T22:00:43.063 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:42 smithi114 bash[63464]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T22:00:43.064 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:42 smithi114 bash[63464]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T22:00:43.064 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:42 smithi114 bash[63464]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T22:00:43.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T22:00:43.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:43.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:43.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:43.825 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:43.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:44.481 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:44 smithi114 bash[63464]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T22:00:44.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:44 smithi114 ceph-mon[23730]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:00:44.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:44 smithi114 ceph-mon[28528]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:00:44.825 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:44 smithi114 bash[63464]: Writing manifest to image destination 2024-09-13T22:00:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:44 smithi121 ceph-mon[24681]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:00:45.235 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:44 smithi114 podman[63464]: 2024-09-13 22:00:44.979676864 +0000 UTC m=+2.720010410 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T22:00:45.235 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 podman[63464]: 2024-09-13 22:00:45.136586421 +0000 UTC m=+2.876919965 container create 43f14e3df9a396aab12500a970c99aa60ec8cc1ed61d93d037731f88417c80e6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:45.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T22:00:45.574 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 podman[63464]: 2024-09-13 22:00:45.355692453 +0000 UTC m=+3.096025990 container init 43f14e3df9a396aab12500a970c99aa60ec8cc1ed61d93d037731f88417c80e6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 podman[63464]: 2024-09-13 22:00:45.358518538 +0000 UTC m=+3.098852078 container start 43f14e3df9a396aab12500a970c99aa60ec8cc1ed61d93d037731f88417c80e6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.361Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.361Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T22:00:45.575 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T22:00:45.576 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T22:00:45.576 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T22:00:45.576 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T22:00:45.576 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T22:00:45.576 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T22:00:45.576 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T22:00:45.577 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T22:00:45.578 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.362Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.363Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a[63715]: ts=2024-09-13T22:00:45.363Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 bash[63464]: 43f14e3df9a396aab12500a970c99aa60ec8cc1ed61d93d037731f88417c80e6 2024-09-13T22:00:45.579 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:00:45 smithi114 systemd[1]: Started Ceph node-exporter.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:00:46.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[23730]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:00:46.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:46.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:46.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[23730]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T22:00:46.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[23730]: Deploying daemon alertmanager.a on smithi114 2024-09-13T22:00:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[28528]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:00:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[28528]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T22:00:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:46 smithi114 ceph-mon[28528]: Deploying daemon alertmanager.a on smithi114 2024-09-13T22:00:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:46 smithi121 ceph-mon[24681]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:00:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:46 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:46 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:46 smithi121 ceph-mon[24681]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T22:00:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:46 smithi121 ceph-mon[24681]: Deploying daemon alertmanager.a on smithi114 2024-09-13T22:00:47.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:47 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:47.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:47 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:47.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:47 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:48.505 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:48] "GET /metrics HTTP/1.1" 200 34910 "" "Prometheus/2.33.4" 2024-09-13T22:00:48.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:48 smithi114 ceph-mon[23730]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:00:48.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:48 smithi114 ceph-mon[28528]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:00:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:48 smithi121 ceph-mon[24681]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:00:50.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:50 smithi114 ceph-mon[28528]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:50.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:50 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:00:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:50 smithi114 ceph-mon[23730]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:50 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:00:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:50 smithi121 ceph-mon[24681]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:50 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:00:52.607 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:00:52 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:00:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.121:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.114:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:00:52.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:52 smithi121 ceph-mon[24681]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:52 smithi114 ceph-mon[23730]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:52 smithi114 ceph-mon[28528]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:53.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=error ts=2024-09-13T22:00:53.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:53.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:53.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.114:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.114 because it doesn't contain any IP SANs" 2024-09-13T22:00:53.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=warn ts=2024-09-13T22:00:53.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.121:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.121 because it doesn't contain any IP SANs" 2024-09-13T22:00:54.761 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:54 smithi114 ceph-mon[23730]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:54.762 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:54 smithi114 ceph-mon[28528]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:54.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:54 smithi121 ceph-mon[24681]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:00:55.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:00:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T22:00:55.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:55 smithi114 systemd[1]: Stopping Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:00:55.734 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[49585]: level=info ts=2024-09-13T22:00:55.440Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-13T22:00:55.734 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:55 smithi114 podman[64170]: 2024-09-13 22:00:55.461035593 +0000 UTC m=+0.161424584 container died 9bf04fad0c856206e4a2aa608704ba029fcb1914e48a55eb7cdff65b62544b05 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:56.067 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:55 smithi114 podman[64170]: 2024-09-13 22:00:55.832403423 +0000 UTC m=+0.532792399 container cleanup 9bf04fad0c856206e4a2aa608704ba029fcb1914e48a55eb7cdff65b62544b05 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:56.067 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:55 smithi114 bash[64170]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T22:00:56.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 podman[64185]: 2024-09-13 22:00:56.066592996 +0000 UTC m=+0.601553338 container remove 9bf04fad0c856206e4a2aa608704ba029fcb1914e48a55eb7cdff65b62544b05 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:56.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 podman[64185]: 2024-09-13 22:00:56.142228202 +0000 UTC m=+0.677188543 volume remove 292785bab4af9a9116296afbfe623664c9d413d2803f9000b988f4ba97741139 2024-09-13T22:00:56.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T22:00:56.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 systemd[1]: Stopped Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:00:56.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.423s CPU time. 2024-09-13T22:00:56.613 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 systemd[1]: Starting Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:00:56.864 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:56 smithi121 ceph-mon[24681]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:57.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:56 smithi114 ceph-mon[23730]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:57.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:56 smithi114 ceph-mon[28528]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:00:57.055 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 podman[64310]: 2024-09-13 22:00:56.634417053 +0000 UTC m=+0.015646067 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-13T22:00:57.055 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 podman[64310]: 2024-09-13 22:00:56.752005483 +0000 UTC m=+0.133234491 volume create 305aa02b1e5be7be82386307ede5f16123a9a0ee58d6948bb3eb576a2486688a 2024-09-13T22:00:57.055 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:56 smithi114 podman[64310]: 2024-09-13 22:00:56.843900975 +0000 UTC m=+0.225129986 container create 4b3ccb3d39923944e11cf8ad2f80a3897b00ddedb1241e11713b56c726e38109 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:57.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 podman[64310]: 2024-09-13 22:00:57.171857113 +0000 UTC m=+0.553086124 container init 4b3ccb3d39923944e11cf8ad2f80a3897b00ddedb1241e11713b56c726e38109 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 podman[64310]: 2024-09-13 22:00:57.174941586 +0000 UTC m=+0.556170594 container start 4b3ccb3d39923944e11cf8ad2f80a3897b00ddedb1241e11713b56c726e38109 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.187Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.187Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.188Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.114 port=9094 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.189Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.212Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.212Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.215Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:57.215Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 bash[64310]: 4b3ccb3d39923944e11cf8ad2f80a3897b00ddedb1241e11713b56c726e38109 2024-09-13T22:00:57.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:57 smithi114 systemd[1]: Started Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:00:57.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:57 smithi121 ceph-mon[24681]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:57.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:57 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:57.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:57 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:57.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:57 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:00:57.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:57 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:00:57.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:57 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:00:57.917 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[23730]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:57.917 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:57.917 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:57.917 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[28528]: from='client.14862 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:00:57.918 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:57 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:00:58.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:00:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:00:58] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-09-13T22:00:58.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:58 smithi114 ceph-mon[23730]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:58.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:58 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:00:58.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:00:58 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:00:58.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:58 smithi114 ceph-mon[28528]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:58.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:58 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:00:58.848 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:00:58 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:00:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:58 smithi121 ceph-mon[24681]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:00:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:58 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:00:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:00:58 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:00:59.442 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:00:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:00:59.190Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000491193s 2024-09-13T22:01:00.694 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:00 smithi114 ceph-mon[23730]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:00.694 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:00 smithi114 ceph-mon[28528]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:00.972 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:00 smithi121 ceph-mon[24681]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:01.338 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:01:01.014+0000 7f331ff09640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:01.338 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Creating ceph-iscsi config... 2024-09-13T22:01:01.338 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:01.338 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:01.338 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:01.338 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return _run_code(code, main_globals, None, 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: exec(code, run_globals) 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:01.339 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.wait_async( 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:01:01.340 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return future.result(timeout) 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.__get_result() 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: raise self._exception 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: out, err, code = await self._run_cephadm( 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: raise OrchestratorError( 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Creating ceph-iscsi config... 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:01.341 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return _run_code(code, main_globals, None, 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: exec(code, run_globals) 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:01.342 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:01.343 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:01.343 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:01.343 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:01.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:01 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:01.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:01 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:01.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:01 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:01.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:01 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:01 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:02.621 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.121:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.114:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[23730]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[28528]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:02.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:02.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:01:02.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:01:02.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:02.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[23730]: Reconfiguring daemon agent.smithi121 on smithi121 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[23730]: Deploying cephadm binary to smithi121 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:01:02.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[28528]: Reconfiguring daemon agent.smithi121 on smithi121 2024-09-13T22:01:02.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:02 smithi114 ceph-mon[28528]: Deploying cephadm binary to smithi121 2024-09-13T22:01:02.917 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:02 smithi121 ceph-mon[24681]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:02.917 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:02 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:02.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:01:02.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:02.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:02.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:02.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:02.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:01:02.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:01:02.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:01:02.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:02 smithi121 ceph-mon[24681]: Reconfiguring daemon agent.smithi121 on smithi121 2024-09-13T22:01:02.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:02 smithi121 ceph-mon[24681]: Deploying cephadm binary to smithi121 2024-09-13T22:01:03.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[28528]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[28528]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[28528]: Deploying daemon prometheus.a on smithi121 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[23730]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:01:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[23730]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:03 smithi114 ceph-mon[23730]: Deploying daemon prometheus.a on smithi121 2024-09-13T22:01:04.077 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:03 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:04.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:03 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:04.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:03 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:04.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:03 smithi121 ceph-mon[24681]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:01:04.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:03 smithi121 ceph-mon[24681]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:04.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:03 smithi121 ceph-mon[24681]: Deploying daemon prometheus.a on smithi121 2024-09-13T22:01:05.510 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:01:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[23959]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:01:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-13T22:01:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:05 smithi121 ceph-mon[24681]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:01:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:05 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:06.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:05 smithi114 ceph-mon[23730]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:01:06.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:05 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:06.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:05 smithi114 ceph-mon[28528]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:01:06.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:05 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:07.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:01:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:01:07.192Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002359989s 2024-09-13T22:01:08.243 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:07 smithi121 ceph-mon[24681]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:01:08.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:07 smithi114 ceph-mon[23730]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:01:08.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:07 smithi114 ceph-mon[28528]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:01:08.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:01:08] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-09-13T22:01:10.200 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:09 smithi121 ceph-mon[24681]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:10.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:09 smithi114 ceph-mon[23730]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:10.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:09 smithi114 ceph-mon[28528]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:12.575 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:12 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.121:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.114:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:12.833 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:12 smithi121 ceph-mon[24681]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:12.833 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:12 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3837174621' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:01:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:12 smithi114 ceph-mon[23730]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:13.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:12 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3837174621' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:01:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:12 smithi114 ceph-mon[28528]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:12 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3837174621' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:01:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:13 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3231394844' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]: dispatch 2024-09-13T22:01:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:13 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]: dispatch 2024-09-13T22:01:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:13 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3231394844' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]: dispatch 2024-09-13T22:01:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:13 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]: dispatch 2024-09-13T22:01:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:13 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3231394844' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]: dispatch 2024-09-13T22:01:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:13 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]: dispatch 2024-09-13T22:01:14.860 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-mon[24681]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:14.861 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]': finished 2024-09-13T22:01:14.861 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-mon[24681]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T22:01:14.861 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2440802541' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3559360226"}]: dispatch 2024-09-13T22:01:14.861 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 systemd[1]: Stopping Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:01:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[23730]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]': finished 2024-09-13T22:01:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[23730]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T22:01:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2440802541' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3559360226"}]: dispatch 2024-09-13T22:01:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[28528]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:01:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2273874582"}]': finished 2024-09-13T22:01:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[28528]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T22:01:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:14 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2440802541' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3559360226"}]: dispatch 2024-09-13T22:01:15.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.859Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T22:01:15.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.859Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.859Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.859Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.859Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.859Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.860Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.860Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.860Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.861Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.861Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[44199]: ts=2024-09-13T22:01:14.861Z caller=main.go:1066 level=info msg="See you next time!" 2024-09-13T22:01:15.122 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:14 smithi121 podman[54381]: 2024-09-13 22:01:14.869990023 +0000 UTC m=+0.115064020 container died baadd3f398d493c26694b92523ce6cdf29b7b85d74bd7484cd47434586a967b9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:01:15.503 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:15 smithi121 podman[54381]: 2024-09-13 22:01:15.285113693 +0000 UTC m=+0.530187690 container cleanup baadd3f398d493c26694b92523ce6cdf29b7b85d74bd7484cd47434586a967b9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:01:15.503 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:15 smithi121 bash[54381]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T22:01:15.760 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:15 smithi121 podman[54396]: 2024-09-13 22:01:15.502882515 +0000 UTC m=+0.634316897 container remove baadd3f398d493c26694b92523ce6cdf29b7b85d74bd7484cd47434586a967b9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:01:15.760 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:15 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2440802541' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3559360226"}]': finished 2024-09-13T22:01:15.760 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:15 smithi121 ceph-mon[24681]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T22:01:15.760 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:15 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/842087637' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]: dispatch 2024-09-13T22:01:15.761 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:15 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]: dispatch 2024-09-13T22:01:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2440802541' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3559360226"}]': finished 2024-09-13T22:01:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[23730]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T22:01:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/842087637' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]: dispatch 2024-09-13T22:01:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]: dispatch 2024-09-13T22:01:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2440802541' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3559360226"}]': finished 2024-09-13T22:01:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[28528]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T22:01:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/842087637' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]: dispatch 2024-09-13T22:01:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:15 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]: dispatch 2024-09-13T22:01:16.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:15 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T22:01:16.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:15 smithi121 systemd[1]: Stopped Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:01:16.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:15 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.277s CPU time. 2024-09-13T22:01:16.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:15 smithi121 systemd[1]: Starting Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:01:16.452 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 podman[54525]: 2024-09-13 22:01:16.087238414 +0000 UTC m=+0.016699117 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T22:01:16.452 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 podman[54525]: 2024-09-13 22:01:16.187203095 +0000 UTC m=+0.116663862 container create 67291dce7d83696b28becfb24ab691782e5205e81fa9731e18cf89b511d57f74 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:01:16.452 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 podman[54525]: 2024-09-13 22:01:16.374559012 +0000 UTC m=+0.304019726 container init 67291dce7d83696b28becfb24ab691782e5205e81fa9731e18cf89b511d57f74 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:01:16.452 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 podman[54525]: 2024-09-13 22:01:16.377245564 +0000 UTC m=+0.306706271 container start 67291dce7d83696b28becfb24ab691782e5205e81fa9731e18cf89b511d57f74 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.401Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.401Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.401Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi121 (none))" 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.402Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.402Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.403Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.403Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.404Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.404Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.406Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.408Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.417866ms 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.408Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.408Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T22:01:16.453 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.427Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.428Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.428Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=38.918µs wal_replay_duration=19.459303ms wbl_replay_duration=104ns total_replay_duration=21.949838ms 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.430Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.430Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.430Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.451Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.902629ms db_storage=1.31µs remote_storage=1.153µs web_handler=468ns query_engine=631ns scrape=450.705µs scrape_sd=125.863µs notify=9.949µs notify_sd=192.175µs rules=19.724553ms tracing=6.171µs 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.451Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T22:01:16.454 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:16.451Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T22:01:16.814 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-mon[24681]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:01:16.815 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]': finished 2024-09-13T22:01:16.815 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-mon[24681]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T22:01:16.815 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3554723550' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]: dispatch 2024-09-13T22:01:16.815 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]: dispatch 2024-09-13T22:01:16.815 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:16.815 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:16 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:16.815 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 bash[54525]: 67291dce7d83696b28becfb24ab691782e5205e81fa9731e18cf89b511d57f74 2024-09-13T22:01:16.815 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:16 smithi121 systemd[1]: Started Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:01:16.975 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[23730]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]': finished 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[23730]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3554723550' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]: dispatch 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]: dispatch 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[28528]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2622975080"}]': finished 2024-09-13T22:01:16.976 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[28528]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T22:01:16.977 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3554723550' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]: dispatch 2024-09-13T22:01:16.977 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]: dispatch 2024-09-13T22:01:16.977 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:16.977 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:16 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[23730]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[23730]: Deploying daemon node-exporter.b on smithi121 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]': finished 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[23730]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2956373252' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3853349338"}]: dispatch 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[28528]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[28528]: Deploying daemon node-exporter.b on smithi121 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]': finished 2024-09-13T22:01:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[28528]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T22:01:17.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:17 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2956373252' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3853349338"}]: dispatch 2024-09-13T22:01:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:17 smithi121 ceph-mon[24681]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T22:01:17.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:17 smithi121 ceph-mon[24681]: Deploying daemon node-exporter.b on smithi121 2024-09-13T22:01:17.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:17 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/209115709"}]': finished 2024-09-13T22:01:17.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:17 smithi121 ceph-mon[24681]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T22:01:17.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:17 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2956373252' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3853349338"}]: dispatch 2024-09-13T22:01:18.458 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:18 smithi121 systemd[1]: Stopping Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:01:18.765 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:18 smithi121 ceph-mon[24681]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T22:01:18.765 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:18 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2956373252' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3853349338"}]': finished 2024-09-13T22:01:18.765 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:18 smithi121 ceph-mon[24681]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T22:01:18.766 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:18 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1058137943' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]: dispatch 2024-09-13T22:01:18.766 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:18 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]: dispatch 2024-09-13T22:01:18.766 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:18 smithi121 podman[54796]: 2024-09-13 22:01:18.457687647 +0000 UTC m=+0.120769110 container died 5452c9d8e4cb3f02ad4a05036d530edf2b62b65ff2cb58bbbf234a55516b15bb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:01:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[23730]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T22:01:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2956373252' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3853349338"}]': finished 2024-09-13T22:01:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[23730]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T22:01:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1058137943' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]: dispatch 2024-09-13T22:01:18.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]: dispatch 2024-09-13T22:01:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[28528]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T22:01:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2956373252' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/3853349338"}]': finished 2024-09-13T22:01:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[28528]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T22:01:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1058137943' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]: dispatch 2024-09-13T22:01:18.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:18 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]: dispatch 2024-09-13T22:01:19.067 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:18 smithi121 podman[54796]: 2024-09-13 22:01:18.874341941 +0000 UTC m=+0.537423400 container cleanup 5452c9d8e4cb3f02ad4a05036d530edf2b62b65ff2cb58bbbf234a55516b15bb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:01:19.067 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:18 smithi121 bash[54796]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T22:01:19.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:19 smithi121 podman[54808]: 2024-09-13 22:01:19.067038586 +0000 UTC m=+0.612970697 container remove 5452c9d8e4cb3f02ad4a05036d530edf2b62b65ff2cb58bbbf234a55516b15bb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:01:19.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:19 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:01:19.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:19 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T22:01:19.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:19 smithi121 systemd[1]: Stopped Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:01:19.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:19 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 4.273s CPU time. 2024-09-13T22:01:19.654 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:19 smithi121 systemd[1]: Starting Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]': finished 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[23730]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3794402211' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]: dispatch 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]: dispatch 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]': finished 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[28528]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3794402211' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]: dispatch 2024-09-13T22:01:19.796 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:19 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]: dispatch 2024-09-13T22:01:20.054 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:19 smithi121 bash[54937]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T22:01:20.054 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:19 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/209115709"}]': finished 2024-09-13T22:01:20.054 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:19 smithi121 ceph-mon[24681]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T22:01:20.054 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:19 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3794402211' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]: dispatch 2024-09-13T22:01:20.054 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:19 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]: dispatch 2024-09-13T22:01:20.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:20 smithi121 bash[54937]: Getting image source signatures 2024-09-13T22:01:20.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:20 smithi121 bash[54937]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T22:01:20.371 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:20 smithi121 bash[54937]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T22:01:20.372 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:20 smithi121 bash[54937]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T22:01:20.941 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[23730]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T22:01:20.942 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]': finished 2024-09-13T22:01:20.942 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[23730]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T22:01:20.942 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:20.942 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[28528]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T22:01:20.942 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]': finished 2024-09-13T22:01:20.942 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[28528]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T22:01:20.942 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:20 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:20 smithi121 ceph-mon[24681]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-13T22:01:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:20 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1177466414"}]': finished 2024-09-13T22:01:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:20 smithi121 ceph-mon[24681]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T22:01:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:20 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:21.871 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:21 smithi121 bash[54937]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T22:01:21.871 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:21 smithi121 bash[54937]: Writing manifest to image destination 2024-09-13T22:01:22.272 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 podman[54937]: 2024-09-13 22:01:21.954555692 +0000 UTC m=+2.369292873 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T22:01:22.273 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 podman[54937]: 2024-09-13 22:01:22.044631524 +0000 UTC m=+2.459368747 container create fa1cf72b5089bf218e0fcf8d58eb18b1faa6e097ba0cee504d0d9ff539003697 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:01:22.273 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 podman[54937]: 2024-09-13 22:01:22.272256614 +0000 UTC m=+2.686993815 container init fa1cf72b5089bf218e0fcf8d58eb18b1faa6e097ba0cee504d0d9ff539003697 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 podman[54937]: 2024-09-13 22:01:22.275138179 +0000 UTC m=+2.689875361 container start fa1cf72b5089bf218e0fcf8d58eb18b1faa6e097ba0cee504d0d9ff539003697 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.278Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.278Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.279Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.279Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.279Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T22:01:22.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T22:01:22.623 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T22:01:22.624 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T22:01:22.625 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T22:01:22.626 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T22:01:22.627 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b[55011]: ts=2024-09-13T22:01:22.280Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T22:01:22.627 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 bash[54937]: fa1cf72b5089bf218e0fcf8d58eb18b1faa6e097ba0cee504d0d9ff539003697 2024-09-13T22:01:22.627 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:01:22 smithi121 systemd[1]: Started Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:01:23.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-mon[24681]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:01:23.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:22 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:01:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[28528]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:01:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[23730]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:22 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:01:23.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:01:23] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-13T22:01:23.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:23 smithi121 ceph-mon[24681]: from='client.24994 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:23.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:23 smithi121 ceph-mon[24681]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T22:01:23.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:23 smithi121 ceph-mon[24681]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T22:01:23.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:23 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:01:23.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:23 smithi121 ceph-mon[24681]: Reconfiguring daemon grafana.a on smithi121 2024-09-13T22:01:23.922 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[23730]: from='client.24994 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:23.922 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[23730]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T22:01:23.922 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[23730]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T22:01:23.922 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:01:23.922 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[23730]: Reconfiguring daemon grafana.a on smithi121 2024-09-13T22:01:23.922 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[28528]: from='client.24994 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:23.923 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[28528]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T22:01:23.923 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[28528]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T22:01:23.923 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:01:23.923 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:23 smithi114 ceph-mon[28528]: Reconfiguring daemon grafana.a on smithi121 2024-09-13T22:01:24.723 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:24 smithi121 ceph-mon[24681]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:24 smithi114 ceph-mon[23730]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:25.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:24 smithi114 ceph-mon[28528]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:26.737 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:26 smithi121 ceph-mon[24681]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 713 B/s rd, 0 op/s 2024-09-13T22:01:27.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:26 smithi114 ceph-mon[23730]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 713 B/s rd, 0 op/s 2024-09-13T22:01:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:26 smithi114 ceph-mon[28528]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 713 B/s rd, 0 op/s 2024-09-13T22:01:28.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:27 smithi121 systemd[1]: Stopping Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:01:28.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[43292]: t=2024-09-13T22:01:27+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-13T22:01:28.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:27 smithi121 podman[55323]: 2024-09-13 22:01:27.922748087 +0000 UTC m=+0.146951485 container died b8bdbef3d4898f0ef5ad05a3d92b443991e80634f7cb55600a9ce1e4d87b6ac5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, io.openshift.tags=base rhel8, io.openshift.expose-services=, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, version=8.5, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vcs-type=git, maintainer=Paul Cuzner ) 2024-09-13T22:01:28.547 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 podman[55323]: 2024-09-13 22:01:28.295375686 +0000 UTC m=+0.519579088 container cleanup b8bdbef3d4898f0ef5ad05a3d92b443991e80634f7cb55600a9ce1e4d87b6ac5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., architecture=x86_64, distribution-scope=public, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, build-date=2022-03-28T10:36:18.413762, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, name=ubi8, vcs-type=git, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , description=Ceph Grafana Container) 2024-09-13T22:01:28.548 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 bash[55323]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a 2024-09-13T22:01:28.548 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 bash[55363]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana.a" found: no such container 2024-09-13T22:01:28.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:28 smithi121 ceph-mon[24681]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:28.871 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 podman[55347]: 2024-09-13 22:01:28.546743224 +0000 UTC m=+0.622732085 container remove b8bdbef3d4898f0ef5ad05a3d92b443991e80634f7cb55600a9ce1e4d87b6ac5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=236.1648460182, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vendor=Red Hat, Inc.) 2024-09-13T22:01:28.872 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T22:01:28.872 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 systemd[1]: Stopped Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:01:28.872 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Consumed 3.283s CPU time. 2024-09-13T22:01:28.872 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:28 smithi121 systemd[1]: Starting Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:01:28.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:28 smithi114 ceph-mon[23730]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:28 smithi114 ceph-mon[28528]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:29.143 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 podman[55482]: 2024-09-13 22:01:29.022889562 +0000 UTC m=+0.344508287 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-09-13T22:01:29.764 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 podman[55482]: 2024-09-13 22:01:29.502636222 +0000 UTC m=+0.824254932 container create 91cdcdf8ef335af58bb828aa13f145c4acfdc6b0928e752c469b3a155a66cc8e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , version=8.5, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-type=git, description=Ceph Grafana Container, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-09-13T22:01:29.764 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 podman[55482]: 2024-09-13 22:01:29.721662366 +0000 UTC m=+1.043281077 container init 91cdcdf8ef335af58bb828aa13f145c4acfdc6b0928e752c469b3a155a66cc8e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, io.openshift.expose-services=, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, vcs-type=git, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, name=ubi8, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.buildah.version=1.24.2) 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 podman[55482]: 2024-09-13 22:01:29.724506571 +0000 UTC m=+1.046125282 container start 91cdcdf8ef335af58bb828aa13f145c4acfdc6b0928e752c469b3a155a66cc8e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, name=ubi8, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public) 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-13T22:01:29.765 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-13T22:01:29.766 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-13T22:01:29.766 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-13T22:01:29.766 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-13T22:01:29.766 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="App mode production" logger=settings 2024-09-13T22:01:29.766 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-13T22:01:29.766 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T22:01:29.766 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-09-13T22:01:30.094 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:29] ENGINE Bus STOPPING 2024-09-13T22:01:30.095 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:01:30.095 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:29] ENGINE Bus STOPPED 2024-09-13T22:01:30.095 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:29] ENGINE Bus STARTING 2024-09-13T22:01:30.095 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Serving on http://:::9283 2024-09-13T22:01:30.095 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=551.854µs 2024-09-13T22:01:30.095 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 bash[55482]: 91cdcdf8ef335af58bb828aa13f145c4acfdc6b0928e752c469b3a155a66cc8e 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 systemd[1]: Started Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-13T22:01:30.096 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:01:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:01:29+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T22:01:30.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STARTED 2024-09-13T22:01:30.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STOPPING 2024-09-13T22:01:30.702 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:01:30.702 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STOPPED 2024-09-13T22:01:30.702 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STARTING 2024-09-13T22:01:30.702 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi114.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:30.703 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:01:30.704 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:30.704 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:01:30.950 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:01:30.950 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.950 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.950 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:01:30.950 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi114.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:01:30.951 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi114.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:30.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:01:30.953 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:01:30.953 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:30 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:01:31.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Serving on http://:::9283 2024-09-13T22:01:31.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STARTED 2024-09-13T22:01:31.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STOPPING 2024-09-13T22:01:31.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:01:31.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STOPPED 2024-09-13T22:01:31.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STARTING 2024-09-13T22:01:31.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Serving on http://:::9283 2024-09-13T22:01:31.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:01:30] ENGINE Bus STARTED 2024-09-13T22:01:31.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:01:31.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi114.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:01:31.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:01:31.752 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:01:31.753 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:01:31.753 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:01:31.753 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: Deploying daemon agent.smithi121 on smithi121 2024-09-13T22:01:31.753 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: Deploying daemon agent.smithi114 on smithi114 2024-09-13T22:01:31.753 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:31.753 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi114.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: Deploying daemon agent.smithi121 on smithi121 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: Deploying daemon agent.smithi114 on smithi114 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:31.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi114.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: Deploying daemon agent.smithi121 on smithi121 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: Deploying daemon agent.smithi114 on smithi114 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:31.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:32.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:32 smithi114 ceph-mon[23730]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-09-13T22:01:32.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:32 smithi114 ceph-mon[28528]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-09-13T22:01:32.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:32 smithi121 ceph-mon[24681]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-09-13T22:01:33.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:01:33] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-13T22:01:33.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='client.24994 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:33.970 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='client.24994 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:01:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='client.24994 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:01:34.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:34.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:34.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:34 smithi121 ceph-mon[24681]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:34.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:34 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:01:34.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:34 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:01:35.015 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:34 smithi114 ceph-mon[28528]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:35.016 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:34 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:01:35.016 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:34 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:01:35.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:34 smithi114 ceph-mon[23730]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:35.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:34 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:01:35.016 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:34 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:01:35.831 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:35.832 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:36.958 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:36 smithi121 ceph-mon[24681]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:37.042 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:36 smithi114 ceph-mon[23730]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:37.042 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:36 smithi114 ceph-mon[28528]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:37.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:37 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:37.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:37 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:37 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:38.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:38 smithi121 ceph-mon[24681]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:38.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:38 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:38.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:38 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:38.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:38 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:38.827 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:39.012 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[23730]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:39.012 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.012 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.012 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.012 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[28528]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:39.013 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.013 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.013 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:38 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-09-13T22:01:39.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:01:39.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:01:39.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:01:39.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:39.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:39 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:39 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:01:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:01:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-09-13T22:01:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1866665340' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:01:41.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T22:01:41.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[23730]: Cluster is now healthy 2024-09-13T22:01:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:01:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:01:41.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-09-13T22:01:41.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1866665340' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:01:41.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T22:01:41.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:40 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T22:01:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:01:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:01:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-09-13T22:01:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:41.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1866665340' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:01:41.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T22:01:41.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:40 smithi121 ceph-mon[24681]: Cluster is now healthy 2024-09-13T22:01:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:42 smithi114 ceph-mon[23730]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:01:43.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:42 smithi114 ceph-mon[28528]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:01:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:42 smithi121 ceph-mon[24681]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:01:43.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:01:43] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.43.0" 2024-09-13T22:01:43.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:43.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:45.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:44 smithi114 ceph-mon[23730]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:44 smithi114 ceph-mon[28528]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:44 smithi121 ceph-mon[24681]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:46 smithi114 ceph-mon[23730]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:47.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:46 smithi114 ceph-mon[28528]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:46 smithi121 ceph-mon[24681]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:48 smithi114 ceph-mon[28528]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:48 smithi114 ceph-mon[23730]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:48 smithi121 ceph-mon[24681]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-09-13T22:01:49.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:50 smithi114 ceph-mon[23730]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 779 B/s rd, 0 op/s 2024-09-13T22:01:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:50 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:50 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:50 smithi114 ceph-mon[28528]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 779 B/s rd, 0 op/s 2024-09-13T22:01:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:50 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:50 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:50 smithi121 ceph-mon[24681]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 779 B/s rd, 0 op/s 2024-09-13T22:01:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:50 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:01:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:50 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:01:52.945 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:52 smithi114 ceph-mon[23730]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:52.945 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:52 smithi114 ceph-mon[28528]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:52 smithi121 ceph-mon[24681]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:53.850 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:01:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:01:53] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-13T22:01:53.850 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:55.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:54 smithi121 ceph-mon[24681]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:55.033 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:54 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.033 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:54 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.033 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:54 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[23730]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:55.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[28528]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:55.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:54 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:56 smithi114 ceph-mon[23730]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:57.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:56 smithi114 ceph-mon[28528]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:56 smithi121 ceph-mon[24681]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:01:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:01:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:01:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:01:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[23730]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-13T22:02:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[23730]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:00.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[23730]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 810 B/s rd, 0 op/s 2024-09-13T22:02:00.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[23730]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:00.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[28528]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-13T22:02:00.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[28528]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:00.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[28528]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 810 B/s rd, 0 op/s 2024-09-13T22:02:00.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:01:59 smithi114 ceph-mon[28528]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:59 smithi121 ceph-mon[24681]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-13T22:02:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:59 smithi121 ceph-mon[24681]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:59 smithi121 ceph-mon[24681]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 810 B/s rd, 0 op/s 2024-09-13T22:02:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:01:59 smithi121 ceph-mon[24681]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:01.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:00 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:01.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:00 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:00 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:02.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:01 smithi114 ceph-mon[23730]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:02.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:01 smithi114 ceph-mon[28528]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:01 smithi121 ceph-mon[24681]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:03.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:02:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:02:03] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-13T22:02:03.745 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:03.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:04.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:03 smithi114 ceph-mon[23730]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:04.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:03 smithi114 ceph-mon[28528]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:03 smithi121 ceph-mon[24681]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:02:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:05 smithi114 ceph-mon[23730]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-09-13T22:02:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:05 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:05 smithi114 ceph-mon[28528]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-09-13T22:02:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:05 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:05 smithi121 ceph-mon[24681]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-09-13T22:02:06.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:05 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:06 smithi114 ceph-mon[23730]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:06 smithi114 ceph-mon[28528]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:06 smithi121 ceph-mon[24681]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:09.066 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:09.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:09 smithi114 ceph-mon[23730]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-09-13T22:02:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:09 smithi114 ceph-mon[28528]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-09-13T22:02:09.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:09 smithi121 ceph-mon[24681]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-09-13T22:02:10.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:10 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:10.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:10 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:10.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:10 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:11.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:11 smithi121 ceph-mon[24681]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:11 smithi114 ceph-mon[23730]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:11.903 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:11 smithi114 ceph-mon[28528]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:13.284 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[28528]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:13.284 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.284 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.285 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[23730]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:13.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:13 smithi121 ceph-mon[24681]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:13.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:13.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:02:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:02:13] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-13T22:02:13.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[23730]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[28528]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:15.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.789 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:15 smithi121 ceph-mon[24681]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:15.789 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.789 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:15.790 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:17 smithi114 ceph-mon[23730]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:18.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:17 smithi114 ceph-mon[28528]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:17 smithi121 ceph-mon[24681]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:19.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:20.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:19 smithi114 ceph-mon[23730]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:20.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:19 smithi114 ceph-mon[28528]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:19 smithi121 ceph-mon[24681]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:20 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:20 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:21.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:20 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:21.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:20 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:20 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:20 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:22.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:21 smithi114 ceph-mon[23730]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:22.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:21 smithi114 ceph-mon[28528]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:21 smithi121 ceph-mon[24681]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:23.872 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:02:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:02:23] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-13T22:02:23.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:23 smithi121 ceph-mon[24681]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:23.873 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:24.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:23 smithi114 ceph-mon[23730]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:24.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:23 smithi114 ceph-mon[28528]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:25 smithi114 ceph-mon[23730]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:26.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:25 smithi114 ceph-mon[28528]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:25 smithi121 ceph-mon[24681]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:28.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:27 smithi114 ceph-mon[23730]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:28.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:27 smithi114 ceph-mon[28528]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:27 smithi121 ceph-mon[24681]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:29.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:29 smithi114 ceph-mon[23730]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:30.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:29 smithi114 ceph-mon[28528]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:29 smithi121 ceph-mon[24681]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:31.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:30 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:31.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:30 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:30 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:32.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:31 smithi114 ceph-mon[23730]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:32.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:31 smithi114 ceph-mon[28528]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:31 smithi121 ceph-mon[24681]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:33.499 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[23730]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:33.499 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.500 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.500 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.500 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[28528]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:33.500 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.500 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.500 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:02:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:02:33] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-13T22:02:33.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:33 smithi121 ceph-mon[24681]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:33.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[23730]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[28528]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:35 smithi121 ceph-mon[24681]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:37.853 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:37 smithi121 ceph-mon[24681]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:37 smithi114 ceph-mon[23730]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:38.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:37 smithi114 ceph-mon[28528]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:39.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:39 smithi114 ceph-mon[23730]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:40.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:39 smithi114 ceph-mon[28528]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:39 smithi121 ceph-mon[24681]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:41.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:40 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:41.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:40 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:40 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:42.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:41 smithi114 ceph-mon[23730]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:41 smithi114 ceph-mon[28528]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:42.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:41 smithi121 ceph-mon[24681]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:43 smithi121 ceph-mon[24681]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:43.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:02:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:02:43] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.43.0" 2024-09-13T22:02:43.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:44.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:43 smithi114 ceph-mon[28528]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:44.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:43 smithi114 ceph-mon[23730]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:46.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:45 smithi114 ceph-mon[23730]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:46.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:45 smithi114 ceph-mon[28528]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:45 smithi121 ceph-mon[24681]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:47 smithi114 ceph-mon[23730]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:48.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:47 smithi114 ceph-mon[28528]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:48.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:47 smithi121 ceph-mon[24681]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:49.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:49 smithi114 ceph-mon[23730]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:49 smithi114 ceph-mon[28528]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:50.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:49 smithi121 ceph-mon[24681]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:51.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:50 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:51.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:50 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:51.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:50 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:51.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:50 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:50 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:02:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:50 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:02:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:51 smithi114 ceph-mon[23730]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:51 smithi114 ceph-mon[28528]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:51 smithi121 ceph-mon[24681]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:53.558 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[23730]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:53.558 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.559 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.559 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.559 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[28528]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:53.559 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.559 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.559 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:02:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:02:53] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T22:02:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:53 smithi121 ceph-mon[24681]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:53.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:55.677 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[28528]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:55.677 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.677 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.677 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.677 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[23730]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:55.678 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.678 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.678 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:55 smithi121 ceph-mon[24681]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:02:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:57.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:57 smithi121 ceph-mon[24681]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:57 smithi114 ceph-mon[23730]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:58.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:57 smithi114 ceph-mon[28528]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:02:59.057 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:58 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:02:59.057 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:58 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:02:59.057 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:58 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:59.057 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:02:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:02:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:02:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:58 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:02:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:58 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:02:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:58 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:02:59.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:58 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:02:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:58 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:02:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:58 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:02:59 smithi114 ceph-mon[23730]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:00.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:02:59 smithi114 ceph-mon[28528]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:02:59 smithi121 ceph-mon[24681]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:01.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:00 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:01.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:00 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:00 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:02.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:01 smithi114 ceph-mon[23730]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:02.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:01 smithi114 ceph-mon[28528]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:01 smithi121 ceph-mon[24681]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:03.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:03 smithi121 ceph-mon[24681]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:03.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:03:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:03:03] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T22:03:03.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:04.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:03 smithi114 ceph-mon[23730]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:04.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:03 smithi114 ceph-mon[28528]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:05.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:05 smithi114 ceph-mon[28528]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:05.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:05 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:05.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:05 smithi114 ceph-mon[23730]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:05.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:05 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:05 smithi121 ceph-mon[24681]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:05 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:07 smithi114 ceph-mon[23730]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:07 smithi114 ceph-mon[28528]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:07 smithi121 ceph-mon[24681]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:09.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:09.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:09 smithi121 ceph-mon[24681]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:09 smithi114 ceph-mon[23730]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:10.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:09 smithi114 ceph-mon[28528]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:10 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:10 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:10 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:11 smithi114 ceph-mon[23730]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:12.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:11 smithi114 ceph-mon[28528]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:11 smithi121 ceph-mon[24681]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:13.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[23730]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[28528]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:03:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:03:13] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T22:03:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:13 smithi121 ceph-mon[24681]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:13.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:15.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[23730]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[28528]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:15.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:15 smithi121 ceph-mon[24681]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:15.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:15.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:17.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:17 smithi121 ceph-mon[24681]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:17 smithi114 ceph-mon[23730]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:18.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:17 smithi114 ceph-mon[28528]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:19.113 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:19.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:19 smithi121 ceph-mon[24681]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:19 smithi114 ceph-mon[23730]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:20.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:19 smithi114 ceph-mon[28528]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:21.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:20 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:21.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:20 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:21.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:20 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:21.060 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:20 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:20 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:20 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:22.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:21 smithi114 ceph-mon[23730]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:21 smithi114 ceph-mon[28528]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:21 smithi121 ceph-mon[24681]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:23.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:23 smithi121 ceph-mon[24681]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:23.876 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:03:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:03:23] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T22:03:23.876 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:24.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:23 smithi114 ceph-mon[23730]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:24.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:23 smithi114 ceph-mon[28528]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:25 smithi114 ceph-mon[23730]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:26.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:25 smithi114 ceph-mon[28528]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:25 smithi121 ceph-mon[24681]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:27.923 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:27 smithi121 ceph-mon[24681]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:28.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:27 smithi114 ceph-mon[23730]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:28.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:27 smithi114 ceph-mon[28528]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:29.100 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:29 smithi114 ceph-mon[23730]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:30.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:29 smithi114 ceph-mon[28528]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:29 smithi121 ceph-mon[24681]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:32.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:31 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:32.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:31 smithi114 ceph-mon[28528]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:31 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:31 smithi114 ceph-mon[23730]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:31 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:31 smithi121 ceph-mon[24681]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:33.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:03:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:03:33] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T22:03:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:33 smithi121 ceph-mon[24681]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:33.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:34.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[28528]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:34.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:34.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[23730]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[28528]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[23730]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:35 smithi121 ceph-mon[24681]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:03:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:37.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:37 smithi114 ceph-mon[23730]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:37.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:37 smithi114 ceph-mon[28528]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:37.886 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T22:03:37.969 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:37 smithi121 ceph-mon[24681]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:03:38.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:39.712 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:39 smithi114 ceph-mon[23730]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:03:39.712 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:39 smithi114 ceph-mon[28528]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:03:39.916 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:39 smithi121 ceph-mon[24681]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:03:40.585 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled to redeploy mgr.y on host 'smithi114' 2024-09-13T22:03:40.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:40 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/146204003' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T22:03:40.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:40 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:03:40.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:40 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:03:40.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.946 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/146204003' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T22:03:40.946 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.946 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.946 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.946 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/146204003' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:03:40.947 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:41.255 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps --refresh' 2024-09-13T22:03:41.926 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:41.926 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:03:41.926 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:03:41.926 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: Schedule redeploy daemon mgr.y 2024-09-13T22:03:41.926 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[23730]: Deploying daemon mgr.y on smithi114 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: Schedule redeploy daemon mgr.y 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:03:41.927 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:03:41.928 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:41 smithi114 ceph-mon[28528]: Deploying daemon mgr.y on smithi114 2024-09-13T22:03:42.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: Schedule redeploy daemon mgr.y 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:03:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:41 smithi121 ceph-mon[24681]: Deploying daemon mgr.y on smithi114 2024-09-13T22:03:43.497 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:03:43.497 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 10s ago 16m - - 2024-09-13T22:03:43.497 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 8s ago 16m - - 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (2m) 10s ago 10m 28.1M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (2m) 8s ago 10m 45.7M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (2m) 10s ago 9m 50.6M - 3.5 e1d6a67b021e db815442a8bc 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283 running (5m) 8s ago 15m 511M - 19.1.1-324-gecab7be5 0652ff2b5d78 56081b4c7747 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:9283 running (17m) 10s ago 17m 410M - 17.2.0 e1d6a67b021e f0c0d3e1c024 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (17m) 10s ago 17m 58.3M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (15m) 8s ago 15m 46.7M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (15m) 10s ago 15m 43.5M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (2m) 10s ago 11m 15.0M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (2m) 8s ago 11m 8280k - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (15m) 10s ago 15m 54.5M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (14m) 10s ago 14m 54.1M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (14m) 10s ago 14m 48.1M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (14m) 10s ago 14m 49.5M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:03:43.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (13m) 8s ago 13m 50.4M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:03:43.499 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (13m) 8s ago 13m 52.0M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:03:43.499 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (12m) 8s ago 12m 48.8M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:03:43.499 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (12m) 8s ago 12m 50.7M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:03:43.499 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (2m) 8s ago 11m 60.6M - 2.43.0 a07b618ecd1d 67291dce7d83 2024-09-13T22:03:43.499 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (10m) 10s ago 10m 72.0M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:03:43.499 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (10m) 8s ago 10m 73.1M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:03:43.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:43 smithi114 ceph-mon[23730]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:43.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:43 smithi114 ceph-mon[28528]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:43 smithi121 ceph-mon[24681]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:43.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:03:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:03:43] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T22:03:43.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:44.163 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T22:03:44.769 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:44 smithi114 ceph-mon[23730]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:03:44.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:44 smithi114 ceph-mon[28528]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:03:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:44 smithi121 ceph-mon[24681]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:03:45.855 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:45 smithi114 ceph-mon[23730]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:45.856 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:45 smithi114 ceph-mon[28528]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:45 smithi121 ceph-mon[24681]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:47.946 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:47 smithi114 ceph-mon[23730]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T22:03:47.946 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:47 smithi114 ceph-mon[28528]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T22:03:48.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:47 smithi121 ceph-mon[24681]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T22:03:49.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:49.976 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:49 smithi114 ceph-mon[23730]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T22:03:49.976 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:49 smithi114 ceph-mon[28528]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T22:03:50.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:49 smithi121 ceph-mon[24681]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T22:03:50.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:50 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:50.979 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:50 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:50.979 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:50 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:03:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:51 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:51 smithi114 ceph-mon[23730]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T22:03:52.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:51 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:51 smithi114 ceph-mon[28528]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T22:03:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:51 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:03:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:51 smithi121 ceph-mon[24681]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-13T22:03:53.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:03:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:03:53] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T22:03:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:53 smithi121 ceph-mon[24681]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T22:03:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:53.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:03:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[23730]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T22:03:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[28528]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-13T22:03:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[23730]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[28528]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:55.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:55 smithi121 ceph-mon[24681]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:03:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:57 smithi114 ceph-mon[23730]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:58.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:57 smithi114 ceph-mon[28528]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:57 smithi121 ceph-mon[24681]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-13T22:03:58.949 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:03:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:03:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:00.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:03:59 smithi121 ceph-mon[24681]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:03:59 smithi114 ceph-mon[23730]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:00.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:03:59 smithi114 ceph-mon[28528]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:01.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:01 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:01.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:01 smithi121 ceph-mon[24681]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:02.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:01 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:02.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:01 smithi114 ceph-mon[23730]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:02.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:01 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:02.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:01 smithi114 ceph-mon[28528]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:03.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:03 smithi121 ceph-mon[24681]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:03.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:04:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:04:03] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T22:04:03.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:03.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:04.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:03 smithi114 ceph-mon[23730]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:04.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:03 smithi114 ceph-mon[28528]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:05 smithi114 ceph-mon[23730]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:05 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:06.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:05 smithi114 ceph-mon[28528]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:05 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:05 smithi121 ceph-mon[24681]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:05 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:07 smithi114 ceph-mon[23730]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:07 smithi114 ceph-mon[28528]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:07 smithi121 ceph-mon[24681]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:08.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:09 smithi114 ceph-mon[23730]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:10.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:09 smithi114 ceph-mon[28528]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:09 smithi121 ceph-mon[24681]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:11 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:11 smithi114 ceph-mon[23730]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:12.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:11 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:12.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:11 smithi114 ceph-mon[28528]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:11 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:11 smithi121 ceph-mon[24681]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:13.737 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:04:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:04:13] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.43.0" 2024-09-13T22:04:13.737 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[23730]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[28528]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:13 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:13 smithi121 ceph-mon[24681]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:13 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[28528]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[23730]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:15 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:15 smithi121 ceph-mon[24681]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:15 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:17 smithi114 ceph-mon[23730]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:18.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:17 smithi114 ceph-mon[28528]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:17 smithi121 ceph-mon[24681]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:19.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:19.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:19 smithi121 ceph-mon[24681]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:19 smithi114 ceph-mon[23730]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:20.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:19 smithi114 ceph-mon[28528]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:20.994 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:20 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:20 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:21.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:20 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:22.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:21 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:22.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:21 smithi114 ceph-mon[23730]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:21 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:21 smithi114 ceph-mon[28528]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:22.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:21 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:22.090 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:21 smithi121 ceph-mon[24681]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:23.760 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:04:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:04:23] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-13T22:04:23.760 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:24.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:23 smithi114 ceph-mon[23730]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:24.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:23 smithi114 ceph-mon[28528]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:24.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:23 smithi121 ceph-mon[24681]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:25 smithi114 ceph-mon[23730]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:26.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:25 smithi114 ceph-mon[28528]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:25 smithi121 ceph-mon[24681]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:27.864 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:27 smithi114 ceph-mon[23730]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:27.865 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:27 smithi114 ceph-mon[28528]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:27 smithi121 ceph-mon[24681]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:29.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:29.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:28 smithi114 systemd[1]: Stopping Ceph mgr.y for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:04:29.512 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:29 smithi114 podman[70448]: 2024-09-13 22:04:29.258536209 +0000 UTC m=+0.134989831 container died f0c0d3e1c0244aea0bc87bd48351d45147159f3693a772b1b0508fa262f065c2 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, architecture=x86_64, RELEASE=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8) 2024-09-13T22:04:29.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:29 smithi114 podman[70448]: 2024-09-13 22:04:29.594060093 +0000 UTC m=+0.470513694 container cleanup f0c0d3e1c0244aea0bc87bd48351d45147159f3693a772b1b0508fa262f065c2 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, distribution-scope=public, architecture=x86_64, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, name=centos-stream, version=8, release=754) 2024-09-13T22:04:29.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:29 smithi114 bash[70448]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y 2024-09-13T22:04:30.021 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:29 smithi114 ceph-mon[23730]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:30.021 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:29 smithi114 ceph-mon[28528]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:30.021 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:29 smithi114 podman[70460]: 2024-09-13 22:04:29.803659126 +0000 UTC m=+0.546747693 container remove f0c0d3e1c0244aea0bc87bd48351d45147159f3693a772b1b0508fa262f065c2 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, release=754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, version=8, GIT_CLEAN=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-type=git, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8) 2024-09-13T22:04:30.021 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:29 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:04:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:29 smithi121 ceph-mon[24681]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:30.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T22:04:30.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 systemd[1]: Stopped Ceph mgr.y for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:04:30.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service: Consumed 1min 6.109s CPU time. 2024-09-13T22:04:30.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 systemd[1]: Starting Ceph mgr.y for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:04:30.693 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 podman[70588]: 2024-09-13 22:04:30.421378325 +0000 UTC m=+0.102808644 container create 8cc35749ec26708bdc9c4dd9d2af19c0f76b8ab8a75d3040599ec5c43f49d125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be) 2024-09-13T22:04:30.694 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 podman[70588]: 2024-09-13 22:04:30.334997526 +0000 UTC m=+0.016427805 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:04:30.694 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 podman[70588]: 2024-09-13 22:04:30.665377132 +0000 UTC m=+0.346807407 container init 8cc35749ec26708bdc9c4dd9d2af19c0f76b8ab8a75d3040599ec5c43f49d125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T22:04:30.694 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 podman[70588]: 2024-09-13 22:04:30.668573525 +0000 UTC m=+0.350003799 container start 8cc35749ec26708bdc9c4dd9d2af19c0f76b8ab8a75d3040599ec5c43f49d125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:04:31.004 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 ceph-mgr[70608]: -- 172.21.15.114:0/446787060 <== mon.1 v2:172.21.15.114:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5627d5eeb0e0 con 0x5627d5ec9000 2024-09-13T22:04:31.004 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 bash[70588]: 8cc35749ec26708bdc9c4dd9d2af19c0f76b8ab8a75d3040599ec5c43f49d125 2024-09-13T22:04:31.004 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 systemd[1]: Started Ceph mgr.y for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:04:31.004 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:30.790+0000 7fe3b1333040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:04:31.004 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:30.856+0000 7fe3b1333040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:04:31.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:31.025+0000 7fe3b1333040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:04:31.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:31.092+0000 7fe3b1333040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:04:31.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:31.256+0000 7fe3b1333040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:04:31.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:31.404+0000 7fe3b1333040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:04:32.001 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:31.737+0000 7fe3b1333040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:31.803+0000 7fe3b1333040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.002 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:04:32.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:04:32.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:32.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:32.000+0000 7fe3b1333040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:04:33.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:32.907+0000 7fe3b1333040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:04:33.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:32.994+0000 7fe3b1333040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:04:33.773 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:04:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:04:33] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-13T22:04:33.773 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:34.046 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[23730]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:34.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.047 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[28528]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:34.047 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.047 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.047 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.048 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:33.777+0000 7fe3b1333040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:04:34.048 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:33.897+0000 7fe3b1333040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:04:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:33 smithi121 ceph-mon[24681]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:33 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:34.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.045+0000 7fe3b1333040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:04:34.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.108+0000 7fe3b1333040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:04:34.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.261+0000 7fe3b1333040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:04:34.614 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.329+0000 7fe3b1333040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:04:34.614 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.389+0000 7fe3b1333040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:04:34.614 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.546+0000 7fe3b1333040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:04:34.614 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.612+0000 7fe3b1333040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:04:34.883 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-mon[23730]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:34.884 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-mon[28528]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:34.884 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.680+0000 7fe3b1333040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:04:34.884 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.762+0000 7fe3b1333040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:04:34.884 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.823+0000 7fe3b1333040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:04:34.884 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:34.882+0000 7fe3b1333040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:04:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:34 smithi121 ceph-mon[24681]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: Standby manager daemon y restarted 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: Standby manager daemon y started 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:04:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: Standby manager daemon y restarted 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: Standby manager daemon y started 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:04:36.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:35.673+0000 7fe3b1333040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:04:35] ENGINE Bus STARTING 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: CherryPy Checker: 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: The Application mounted at '' has an empty config. 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:04:35] ENGINE Serving on http://:::9283 2024-09-13T22:04:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:04:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:04:35] ENGINE Bus STARTED 2024-09-13T22:04:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: Standby manager daemon y restarted 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: Standby manager daemon y started 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:04:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:35 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:04:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:37 smithi114 ceph-mon[23730]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:37 smithi114 ceph-mon[23730]: mgrmap e26: x(active, since 4m), standbys: y 2024-09-13T22:04:38.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:37 smithi114 ceph-mon[28528]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:38.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:37 smithi114 ceph-mon[28528]: mgrmap e26: x(active, since 4m), standbys: y 2024-09-13T22:04:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:37 smithi121 ceph-mon[24681]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:37 smithi121 ceph-mon[24681]: mgrmap e26: x(active, since 4m), standbys: y 2024-09-13T22:04:39.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:39.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:39 smithi114 ceph-mon[23730]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:39.932 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:39 smithi114 ceph-mon[28528]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:39 smithi121 ceph-mon[24681]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:41.801 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:41 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:41.801 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:41 smithi121 ceph-mon[24681]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:42.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:41 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:41 smithi114 ceph-mon[23730]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:41 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:41 smithi114 ceph-mon[28528]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:43.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:43 smithi121 ceph-mon[24681]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:43.872 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:04:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:04:43] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-13T22:04:43.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:44.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:43 smithi114 ceph-mon[23730]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:43 smithi114 ceph-mon[28528]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:46.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:45 smithi114 ceph-mon[23730]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:46.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:45 smithi114 ceph-mon[28528]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:46.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:45 smithi121 ceph-mon[24681]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:47.934 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:47 smithi114 ceph-mon[23730]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:47.934 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:47 smithi114 ceph-mon[28528]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:48.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:47 smithi121 ceph-mon[24681]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:49.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:49.804 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:49 smithi114 ceph-mon[23730]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:49.804 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:49 smithi114 ceph-mon[28528]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:50.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:49 smithi121 ceph-mon[24681]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:50.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:50 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:50.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:50 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:50 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:04:51.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:51 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:51.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:51 smithi114 ceph-mon[23730]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:51.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:51 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:51.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:51 smithi114 ceph-mon[28528]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:51 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:04:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:51 smithi121 ceph-mon[24681]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:53.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:04:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:04:53] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-13T22:04:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:53 smithi121 ceph-mon[24681]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:53.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:53 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:53.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:04:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[23730]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[28528]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:54.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:53 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:55 smithi121 ceph-mon[24681]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:55.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:55.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:55 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:56.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[23730]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:56.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:56.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:56.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[28528]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:04:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:55 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:04:57.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:57 smithi114 ceph-mon[23730]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:57.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:57 smithi114 ceph-mon[28528]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:57 smithi121 ceph-mon[24681]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:04:58.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:04:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:04:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:04:59 smithi114 ceph-mon[23730]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:00.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:04:59 smithi114 ceph-mon[28528]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:04:59 smithi121 ceph-mon[24681]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:02.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:01 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:02.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:01 smithi114 ceph-mon[28528]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:02.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:01 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:02.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:01 smithi114 ceph-mon[23730]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:01 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:01 smithi121 ceph-mon[24681]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:03.706 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:05:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:05:03] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-13T22:05:03.706 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:03.966 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:03 smithi121 ceph-mon[24681]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:03.986 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:03 smithi114 ceph-mon[23730]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:03.987 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:03 smithi114 ceph-mon[28528]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:06.039 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[23730]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:06.039 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:06.039 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.039 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.039 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.040 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[28528]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:06.040 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:06.040 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.040 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.040 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:05 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:05 smithi121 ceph-mon[24681]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:05 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:05 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:05 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:05 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:07 smithi114 ceph-mon[23730]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:07 smithi114 ceph-mon[28528]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:07 smithi121 ceph-mon[24681]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:09.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:09 smithi114 ceph-mon[23730]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:10.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:09 smithi114 ceph-mon[28528]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:09 smithi121 ceph-mon[24681]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:12.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:11 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:12.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:11 smithi114 ceph-mon[28528]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:11 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:11 smithi114 ceph-mon[23730]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:11 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:11 smithi121 ceph-mon[24681]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:13 smithi121 ceph-mon[24681]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:13.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:05:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:05:13] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-13T22:05:13.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:13 smithi114 ceph-mon[23730]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:13 smithi114 ceph-mon[28528]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:15 smithi114 ceph-mon[23730]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:15 smithi114 ceph-mon[28528]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:15 smithi121 ceph-mon[24681]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:16 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:16 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:16 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:16 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:16 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:16 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:16 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:16 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:16 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:17 smithi114 ceph-mon[23730]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:18.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:17 smithi114 ceph-mon[28528]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:17 smithi121 ceph-mon[24681]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:19.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[23730]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[28528]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:19 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:19 smithi121 ceph-mon[24681]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:19 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:19 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:19 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:20 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:21.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:20 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:20 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:21 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:21 smithi114 ceph-mon[28528]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:22.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:21 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:22.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:21 smithi114 ceph-mon[23730]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:22.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:21 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:22.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:21 smithi121 ceph-mon[24681]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:23.754 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:05:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:05:23] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T22:05:23.755 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:23.977 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:23 smithi114 ceph-mon[23730]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:23.977 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:23 smithi114 ceph-mon[28528]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:24.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:23 smithi121 ceph-mon[24681]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:26.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:25 smithi114 ceph-mon[23730]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:26.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:25 smithi114 ceph-mon[28528]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:26.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:25 smithi121 ceph-mon[24681]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:28.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:27 smithi114 ceph-mon[23730]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:28.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:27 smithi114 ceph-mon[28528]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:27 smithi121 ceph-mon[24681]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:29.023 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:29 smithi114 ceph-mon[23730]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:30.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:29 smithi114 ceph-mon[28528]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:29 smithi121 ceph-mon[24681]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:32.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:32.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[23730]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:32.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:05:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:05:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[28528]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:05:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:05:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:31 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:31 smithi121 ceph-mon[24681]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:31 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:05:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:31 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:05:32.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:33.768 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:05:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:05:33] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-13T22:05:33.769 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:34.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:33 smithi114 ceph-mon[23730]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:34.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:33 smithi114 ceph-mon[28528]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:33 smithi121 ceph-mon[24681]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:36.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:35 smithi114 ceph-mon[28528]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:36.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:35 smithi114 ceph-mon[23730]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:35 smithi121 ceph-mon[24681]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:36 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:36 smithi121 ceph-mon[24681]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:37.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[28528]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:36 smithi114 ceph-mon[23730]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:39.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:39.396 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:39 smithi114 ceph-mon[23730]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:39.396 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:39 smithi114 ceph-mon[28528]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:39 smithi121 ceph-mon[24681]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:40 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:40 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:41.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:40 smithi121 ceph-mon[24681]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:41.182 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.182 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.182 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.182 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:41.182 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[23730]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:41.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:41.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:41.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:40 smithi114 ceph-mon[28528]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:43.295 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:43 smithi121 ceph-mon[24681]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:43.566 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:05:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:05:43] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.43.0" 2024-09-13T22:05:43.566 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:43.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:43 smithi114 ceph-mon[23730]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:43.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:43 smithi114 ceph-mon[28528]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:45.308 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:45 smithi121 ceph-mon[24681]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:45 smithi114 ceph-mon[23730]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:45 smithi114 ceph-mon[28528]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:47.943 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:47 smithi114 ceph-mon[28528]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:47.943 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:47 smithi114 ceph-mon[23730]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:48.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:47 smithi121 ceph-mon[24681]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:49.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:49 smithi114 ceph-mon[23730]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:49 smithi114 ceph-mon[28528]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:50.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:49 smithi121 ceph-mon[24681]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:51.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:50 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:50 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:50 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:05:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:51 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:51 smithi114 ceph-mon[23730]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:51 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:51 smithi114 ceph-mon[28528]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:51 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:05:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:51 smithi121 ceph-mon[24681]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:53.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:53 smithi121 ceph-mon[24681]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:53.872 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:05:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:05:53] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.43.0" 2024-09-13T22:05:53.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:53 smithi114 ceph-mon[23730]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:54.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:53 smithi114 ceph-mon[28528]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:56.002 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:55 smithi121 ceph-mon[24681]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:56.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:55 smithi114 ceph-mon[23730]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:56.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:55 smithi114 ceph-mon[28528]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[23730]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.246 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:56 smithi114 ceph-mon[28528]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:56 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:56 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:56 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:05:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:56 smithi121 ceph-mon[24681]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:05:59.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:05:58 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:05:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"172.21.15.121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.114\", device_class=\"ssd\", hostname=\"smithi114\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.114\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:05:59.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:05:59 smithi114 ceph-mon[23730]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:59.562 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:05:59 smithi114 ceph-mon[28528]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:05:59.571 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:05:59 smithi121 ceph-mon[24681]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:01.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:01.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[23730]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:00 smithi114 ceph-mon[28528]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:00 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:00 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:00 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:00 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:00 smithi121 ceph-mon[24681]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:03.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:03 smithi121 ceph-mon[24681]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:03.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:03 smithi114 ceph-mon[23730]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:03.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:03 smithi114 ceph-mon[28528]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:03.796 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:06:03] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.43.0" 2024-09-13T22:06:03.797 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:06:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:06:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi114\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8ccc9cd8-7219-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi114\", job=\"node\", machine=\"x86_64\", nodename=\"smithi114\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T22:06:05.369 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:05 smithi121 ceph-mon[24681]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:05.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:05 smithi114 ceph-mon[23730]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:05.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:05 smithi114 ceph-mon[28528]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:06.478 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:06 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:06.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:06 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:06.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:06 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:07 smithi114 ceph-mon[23730]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:08.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:07 smithi114 ceph-mon[28528]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:07 smithi121 ceph-mon[24681]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:09 smithi114 ceph-mon[23730]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:10.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:09 smithi114 ceph-mon[28528]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:09 smithi121 ceph-mon[24681]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:11 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:11 smithi114 ceph-mon[23730]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:12.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:11 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:12.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:11 smithi114 ceph-mon[28528]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:12.081 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:11 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:12.081 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:11 smithi121 ceph-mon[24681]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:13 smithi121 ceph-mon[24681]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:13.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:06:13] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-13T22:06:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:13 smithi114 ceph-mon[23730]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:13 smithi114 ceph-mon[28528]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:15 smithi114 ceph-mon[23730]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:15 smithi114 ceph-mon[28528]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:15 smithi121 ceph-mon[24681]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:17 smithi121 ceph-mon[24681]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:17 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:17 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:17 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[23730]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:17.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[28528]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:17 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:19.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:19 smithi114 ceph-mon[23730]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:19.264 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:19 smithi114 ceph-mon[28528]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:19.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:19 smithi121 ceph-mon[24681]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:21 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:21.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:21 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:21 smithi121 ceph-mon[24681]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:21.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:21 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:21 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:21 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[23730]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[28528]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:21.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:21.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:21 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:23 smithi121 ceph-mon[24681]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:23.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:23 smithi114 ceph-mon[23730]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:23.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:23 smithi114 ceph-mon[28528]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:23.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:06:23] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-13T22:06:25.179 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:25 smithi121 ceph-mon[24681]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:25.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:25 smithi114 ceph-mon[23730]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:25.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:25 smithi114 ceph-mon[28528]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:28.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:27 smithi114 ceph-mon[23730]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:27 smithi114 ceph-mon[28528]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:27 smithi121 ceph-mon[24681]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:29 smithi114 ceph-mon[23730]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:30.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:29 smithi114 ceph-mon[28528]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:29 smithi121 ceph-mon[24681]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[23730]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[28528]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:06:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:31 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:31 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:31 smithi121 ceph-mon[24681]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:31 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:06:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:31 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:06:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:31 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:33 smithi121 ceph-mon[24681]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:33.872 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:06:33] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-13T22:06:34.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:33 smithi114 ceph-mon[23730]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:34.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:33 smithi114 ceph-mon[28528]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:36.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:35 smithi114 ceph-mon[23730]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:35 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:35 smithi114 ceph-mon[28528]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:35 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:35 smithi121 ceph-mon[24681]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:35 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[23730]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[28528]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:37 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:37 smithi121 ceph-mon[24681]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:37.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:37 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:37 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:37.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:37 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:39.538 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:39 smithi114 ceph-mon[23730]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:39.539 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:39 smithi114 ceph-mon[28528]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:39 smithi121 ceph-mon[24681]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:41.531 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[23730]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[28528]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.532 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:41 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:41 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:41.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:41 smithi121 ceph-mon[24681]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:41.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:41 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:41 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:41.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:41 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:43.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:43 smithi121 ceph-mon[24681]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:43.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:43 smithi114 ceph-mon[23730]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:43.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:43 smithi114 ceph-mon[28528]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:43.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:06:43] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.43.0" 2024-09-13T22:06:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:45 smithi114 ceph-mon[23730]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:45 smithi114 ceph-mon[28528]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:45.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:45 smithi121 ceph-mon[24681]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:46.827 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:06:48.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:47 smithi114 ceph-mon[23730]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:48.008 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:47 smithi114 ceph-mon[28528]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:48.071 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:47 smithi121 ceph-mon[24681]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 8s ago 19m - - 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 12s ago 19m - - 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (5m) 8s ago 14m 26.6M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (5m) 12s ago 13m 56.3M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (5m) 8s ago 12m 53.5M - 3.5 e1d6a67b021e db815442a8bc 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283 running (9m) 12s ago 18m 516M - 19.1.1-324-gecab7be5 0652ff2b5d78 56081b4c7747 2024-09-13T22:06:48.719 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (2m) 8s ago 20m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:06:48.720 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (20m) 8s ago 20m 65.2M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:06:48.720 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (18m) 12s ago 18m 52.6M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:06:48.720 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (19m) 8s ago 19m 49.2M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:06:48.720 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (6m) 8s ago 14m 14.7M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:06:48.720 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (5m) 12s ago 14m 14.7M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:06:48.720 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (18m) 8s ago 18m 55.7M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:06:48.720 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (18m) 8s ago 17m 55.2M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:06:48.721 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (17m) 8s ago 17m 49.8M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:06:48.721 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (17m) 8s ago 17m 49.8M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:06:48.721 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (16m) 12s ago 16m 51.7M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:06:48.721 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (16m) 12s ago 16m 53.0M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:06:48.721 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (16m) 12s ago 16m 50.0M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:06:48.721 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (15m) 12s ago 15m 51.7M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:06:48.722 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (5m) 12s ago 14m 67.1M - 2.43.0 a07b618ecd1d 67291dce7d83 2024-09-13T22:06:48.722 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (13m) 8s ago 13m 73.0M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:06:48.722 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (13m) 12s ago 13m 74.0M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:06:49.254 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T22:06:49.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:49 smithi114 ceph-mon[23730]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:49.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:49 smithi114 ceph-mon[23730]: from='client.25099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:06:49.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:49 smithi114 ceph-mon[28528]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:49.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:49 smithi114 ceph-mon[28528]: from='client.25099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:06:50.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:49 smithi121 ceph-mon[24681]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:50.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:49 smithi121 ceph-mon[24681]: from='client.25099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:06:51.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:50 smithi114 ceph-mon[23730]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:51.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:50 smithi114 ceph-mon[28528]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:50 smithi121 ceph-mon[24681]: from='mgr.24791 172.21.15.121:0/1805065358' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:06:51.184 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:06:51.185 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:06:51.777 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T22:06:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:51 smithi114 ceph-mon[23730]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:51 smithi114 ceph-mon[23730]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:51 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1603890735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:06:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:51 smithi114 ceph-mon[28528]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:51 smithi114 ceph-mon[28528]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:51 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1603890735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:06:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:51 smithi121 ceph-mon[24681]: from='client.25060 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:06:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:51 smithi121 ceph-mon[24681]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:51 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1603890735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:06:53.799 INFO:teuthology.orchestra.run.smithi114.stdout:HEALTH_OK 2024-09-13T22:06:53.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:53 smithi121 ceph-mon[24681]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:53.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:06:53] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-13T22:06:54.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:53 smithi114 ceph-mon[28528]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:54.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:53 smithi114 ceph-mon[23730]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:54.452 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T22:06:54.970 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:54 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2346323876' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:06:54.970 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:54 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2346323876' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:06:55.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:54 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2346323876' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:06:56.044 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:55 smithi114 ceph-mon[23730]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:56.044 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:55 smithi114 ceph-mon[28528]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:55 smithi121 ceph-mon[24681]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:56.436 INFO:teuthology.orchestra.run.smithi114.stdout: cluster: 2024-09-13T22:06:56.436 INFO:teuthology.orchestra.run.smithi114.stdout: id: 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: health: HEALTH_OK 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: services: 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: mgr: x(active, since 6m), standbys: y 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: data: 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: pools: 6 pools, 161 pgs 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: objects: 209 objects, 585 KiB 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:06:56.437 INFO:teuthology.orchestra.run.smithi114.stdout: pgs: 161 active+clean 2024-09-13T22:06:56.438 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:06:56.438 INFO:teuthology.orchestra.run.smithi114.stdout: io: 2024-09-13T22:06:56.438 INFO:teuthology.orchestra.run.smithi114.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-13T22:06:56.438 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:06:57.013 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr fail' 2024-09-13T22:06:57.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[23730]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:57.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[23730]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3977283067' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:06:57.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[28528]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:57.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[28528]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:57 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3977283067' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:06:57.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:57 smithi121 ceph-mon[24681]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:06:57.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:57 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:57 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:57 smithi121 ceph-mon[24681]: from='mgr.24791 ' entity='mgr.x' 2024-09-13T22:06:57.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:57 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3977283067' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:06:59.641 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[23730]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:59.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3837032031' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:06:59.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:06:59.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[23730]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T22:06:59.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[28528]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:59.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3837032031' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:06:59.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:06:59.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-mon[28528]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T22:06:59.645 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:06:59] ENGINE Bus STOPPING 2024-09-13T22:06:59.645 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:06:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:06:59.645 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:06:59] ENGINE Bus STOPPED 2024-09-13T22:06:59.744 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-mon[24681]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:06:59.744 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3837032031' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:06:59.744 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:06:59.744 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-mon[24681]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T22:06:59.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:06:59.384+0000 7f338ac3c640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T22:06:59.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ignoring --setuser ceph since I am not root 2024-09-13T22:06:59.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ignoring --setgroup ceph since I am not root 2024-09-13T22:06:59.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-mgr[49218]: -- 172.21.15.121:0/1505178642 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56460bd1f0e0 con 0x56460bcfc800 2024-09-13T22:06:59.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:06:59.536+0000 7fb892558040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:06:59.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:06:59.593+0000 7fb892558040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:06:59.745 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:06:59.743+0000 7fb892558040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:07:00.039 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:06:59] ENGINE Bus STARTING 2024-09-13T22:07:00.039 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:06:59] ENGINE Serving on http://:::9283 2024-09-13T22:07:00.039 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:06:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:06:59] ENGINE Bus STARTED 2024-09-13T22:07:00.062 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T22:07:00.101 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:06:59.809+0000 7fb892558040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:07:00.101 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:06:59 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:06:59.960+0000 7fb892558040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:07:00.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:00.100+0000 7fb892558040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:07:00.552 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:07:00.552 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: mgrmap e27: y(active, starting, since 0.33227s) 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:07:00.553 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: Manager daemon y is now available 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:07:00.554 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: mgrmap e27: y(active, starting, since 0.33227s) 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:07:00.555 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: Manager daemon y is now available 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:00.556 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:07:00.557 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:07:00.557 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:07:00.557 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:07:00.557 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:00.557 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:00.557 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:07:00.557 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:07:00.558 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:00.558 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:00 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:00.659 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:00.430+0000 7fb892558040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:07:00.659 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:00.495+0000 7fb892558040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: mgrmap e27: y(active, starting, since 0.33227s) 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:07:00.660 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: Manager daemon y is now available 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:07:00.661 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:00.662 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:07:00.662 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:07:00.662 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:07:00.662 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:07:00.662 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:00.662 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:01.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:00 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:00.693+0000 7fb892558040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:07:01.427 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[23730]: mgrmap e28: y(active, since 1.345s) 2024-09-13T22:07:01.427 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:01.427 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:01.427 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[28528]: mgrmap e28: y(active, since 1.345s) 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:01.428 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:07:01.429 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:07:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-mon[24681]: mgrmap e28: y(active, since 1.345s) 2024-09-13T22:07:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:07:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:07:01.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:07:02.023 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:01.675+0000 7fb892558040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:07:02.023 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:01.749+0000 7fb892558040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:07:02.248 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:07:01.828+0000 7fe3092a4640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:22:07:01] ENGINE Error in HTTPServer.serve 2024-09-13T22:07:02.248 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._connections.run(self.expiration_interval) 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._run(expiration_interval) 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: s = self.context.wrap_socket( 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.sslsocket_class._create( 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:07:02.249 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.do_handshake() 2024-09-13T22:07:02.250 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:07:02.250 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._sslobj.do_handshake() 2024-09-13T22:07:02.250 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:07:02.250 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:07:02.539 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:07:02.539 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:07:02.539 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:07:02.539 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:07:02.539 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: [13/Sep/2024:22:07:01] ENGINE Bus STARTING 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:02.540 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: [13/Sep/2024:22:07:01] ENGINE Bus STARTING 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.542 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:02 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.704 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:02.524+0000 7fb892558040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:02.649+0000 7fb892558040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.705 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: [13/Sep/2024:22:07:01] ENGINE Bus STARTING 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:02.706 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:02.708 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:02.708 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.708 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:02.993 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:02.801+0000 7fb892558040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:07:02.993 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:02.864+0000 7fb892558040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:07:03.298 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.034+0000 7fb892558040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:07:03.298 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.114+0000 7fb892558040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:07:03.298 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.179+0000 7fb892558040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:07:03.593 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.364+0000 7fb892558040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:07:03.593 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.433+0000 7fb892558040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:07:03.593 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.503+0000 7fb892558040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:07:03.593 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.591+0000 7fb892558040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:07:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:07:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:07:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: [13/Sep/2024:22:07:01] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T22:07:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:07:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: [13/Sep/2024:22:07:01] ENGINE Error in HTTPServer.serve 2024-09-13T22:07:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:03.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.do_handshake() 2024-09-13T22:07:03.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: [13/Sep/2024:22:07:01] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: [13/Sep/2024:22:07:01] ENGINE Bus STARTED 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: mgrmap e29: y(active, since 3s) 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: [13/Sep/2024:22:07:01] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: [13/Sep/2024:22:07:01] ENGINE Error in HTTPServer.serve 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:07:03.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.do_handshake() 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:07:03.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: [13/Sep/2024:22:07:01] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: [13/Sep/2024:22:07:01] ENGINE Bus STARTED 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: mgrmap e29: y(active, since 3s) 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.830 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:07:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:07:03.511Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:07:03.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.655+0000 7fb892558040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:07:03.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:03.719+0000 7fb892558040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:07:03.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:07:03.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:07:03.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: [13/Sep/2024:22:07:01] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: [13/Sep/2024:22:07:01] ENGINE Error in HTTPServer.serve 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._run(expiration_interval) 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:07:03.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: s = self.context.wrap_socket( 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.sslsocket_class._create( 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.do_handshake() 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._sslobj.do_handshake() 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: [13/Sep/2024:22:07:01] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: [13/Sep/2024:22:07:01] ENGINE Bus STARTED 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: mgrmap e29: y(active, since 3s) 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:03.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:03 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.594 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:04.541+0000 7fb892558040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:07:04.594 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:07:04] ENGINE Bus STARTING 2024-09-13T22:07:04.594 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: CherryPy Checker: 2024-09-13T22:07:04.594 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: The Application mounted at '' has an empty config. 2024-09-13T22:07:04.594 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:07:04.799 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.800 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.801 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T22:07:04.801 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:07:04.801 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:07:04.801 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:07:04.801 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:07:04.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:07:04] ENGINE Serving on http://:::9283 2024-09-13T22:07:04.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:07:04] ENGINE Bus STARTED 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:07:04.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:04 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:07:05.064 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:07:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:07:04.990Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=3 2024-09-13T22:07:05.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:05.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:05.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:05.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: mgrmap e30: y(active, since 6s), standbys: x 2024-09-13T22:07:05.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:07:05.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:05.811 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:05.812 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:05 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: mgrmap e30: y(active, since 6s), standbys: x 2024-09-13T22:07:06.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: mgrmap e30: y(active, since 6s), standbys: x 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.014 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:05 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:06.793 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:06 smithi114 ceph-mon[23730]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:06.794 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:06 smithi114 ceph-mon[28528]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:06 smithi121 ceph-mon[24681]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:07:08.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:07:08.028+0000 7fe341591640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:08.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:08.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:08.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:08.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:08.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:08.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:08.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:08.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:07:08.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:08.217 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:08.218 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:08.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[23730]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:07:08.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[28528]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:07:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.930 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:08 smithi121 ceph-mon[24681]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:07:08.931 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.931 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:08.931 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:09.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:09 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:09.824 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:07:09.824 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:07:09.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:09 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:07:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:07:09.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:07:09.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:07:09.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:07:09.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:07:09.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:07:09.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:07:09.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:07:09.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:09 smithi114 ceph-mon[23730]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:09 smithi114 ceph-mon[23730]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:09.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:09.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:09.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:09 smithi114 ceph-mon[28528]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:07:09.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:09 smithi114 ceph-mon[28528]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:07:10.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:09 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:10.006 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:07:10.006 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:10.007 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:07:10.008 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:07:10.009 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:07:10.010 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:07:10.011 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:07:10.011 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:07:10.011 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:07:10.011 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:07:10.011 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:07:10.011 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:09 smithi121 ceph-mon[24681]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:07:10.011 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:09 smithi121 ceph-mon[24681]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:07:11.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:10 smithi121 ceph-mon[24681]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:07:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:10 smithi114 ceph-mon[23730]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:07:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:10 smithi114 ceph-mon[28528]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:07:11.621 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 systemd[1]: Stopping Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:07:11.996 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T22:07:11.996 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T22:07:11.996 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T22:07:11.996 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T22:07:11.996 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T22:07:11.996 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T22:07:11.996 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T22:07:11.997 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.720Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T22:07:11.997 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.721Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T22:07:11.997 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.721Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T22:07:11.997 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.721Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T22:07:11.997 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[54540]: ts=2024-09-13T22:07:11.721Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T22:07:11.997 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:11 smithi121 podman[62129]: 2024-09-13 22:07:11.730750376 +0000 UTC m=+0.133459946 container died 67291dce7d83696b28becfb24ab691782e5205e81fa9731e18cf89b511d57f74 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:07:12.371 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 podman[62129]: 2024-09-13 22:07:12.103156012 +0000 UTC m=+0.505865579 container cleanup 67291dce7d83696b28becfb24ab691782e5205e81fa9731e18cf89b511d57f74 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:07:12.371 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 bash[62129]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T22:07:12.697 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 podman[62141]: 2024-09-13 22:07:12.383573149 +0000 UTC m=+0.654551361 container remove 67291dce7d83696b28becfb24ab691782e5205e81fa9731e18cf89b511d57f74 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:07:12.697 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T22:07:12.697 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 systemd[1]: Stopped Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:07:12.697 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.131s CPU time. 2024-09-13T22:07:12.697 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 systemd[1]: Starting Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:07:13.008 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:12 smithi121 ceph-mon[24681]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:07:13.008 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 podman[62237]: 2024-09-13 22:07:12.643979004 +0000 UTC m=+0.016968930 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T22:07:13.008 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:12 smithi121 podman[62237]: 2024-09-13 22:07:12.772482915 +0000 UTC m=+0.145472830 container create eba628b49c89c89334d50f1b2396929a2e145618c486d6c3c750fd2eafa4bb0a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:07:13.009 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 podman[62237]: 2024-09-13 22:07:13.007918939 +0000 UTC m=+0.380908854 container init eba628b49c89c89334d50f1b2396929a2e145618c486d6c3c750fd2eafa4bb0a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:07:13.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:12 smithi114 ceph-mon[28528]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:07:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:12 smithi114 ceph-mon[23730]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:07:13.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 podman[62237]: 2024-09-13 22:07:13.010694635 +0000 UTC m=+0.383684559 container start eba628b49c89c89334d50f1b2396929a2e145618c486d6c3c750fd2eafa4bb0a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:07:13.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.034Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T22:07:13.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.034Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T22:07:13.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.034Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi121 (none))" 2024-09-13T22:07:13.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.034Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.034Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.036Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.036Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.037Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.037Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.038Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.041Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.691249ms 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.041Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.041Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T22:07:13.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.058Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.071Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.071Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.071Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=64.37µs wal_replay_duration=29.754666ms wbl_replay_duration=112ns total_replay_duration=32.556731ms 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.074Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.074Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.074Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.093Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.312542ms db_storage=840ns remote_storage=1.35µs web_handler=542ns query_engine=673ns scrape=420.087µs scrape_sd=96.175µs notify=10.627µs notify_sd=14.457µs rules=17.408068ms tracing=6.637µs 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.093Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:07:13.093Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 bash[62237]: eba628b49c89c89334d50f1b2396929a2e145618c486d6c3c750fd2eafa4bb0a 2024-09-13T22:07:13.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:07:13 smithi121 systemd[1]: Started Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:07:14.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:07:14.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:07:14.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:14.311 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:07:14.312 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:14.313 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:14.313 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:14.313 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:14 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:07:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:07:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:07:14.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:14.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:14.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:14.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:14.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:07:14.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:14.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:07:14.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:14.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:14 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:07:15.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:15 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:15 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:15 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:17.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2363382140' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:07:17.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:07:17.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:17.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:17.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:16 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2363382140' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:07:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:07:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:07:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:07:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:07:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:16 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2363382140' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:07:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:17 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:07:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:17 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:07:18.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:17 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:07:18.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:17 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:07:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:17 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:07:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:17 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:07:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:18 smithi114 ceph-mon[23730]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:19.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:18 smithi114 ceph-mon[28528]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:18 smithi121 ceph-mon[24681]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:07:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:20 smithi114 ceph-mon[23730]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:07:21.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:20 smithi114 ceph-mon[28528]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:07:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:20 smithi121 ceph-mon[24681]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:07:22.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:07:21] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-09-13T22:07:23.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:22 smithi114 ceph-mon[23730]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T22:07:23.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:22 smithi114 ceph-mon[28528]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T22:07:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:22 smithi121 ceph-mon[24681]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-09-13T22:07:24.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:24 smithi121 ceph-mon[24681]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:07:25.006 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:24 smithi114 ceph-mon[23730]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:07:25.006 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:24 smithi114 ceph-mon[28528]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:07:26.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:26 smithi121 ceph-mon[24681]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:07:26.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:26 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:26 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:26 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[28528]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:07:26.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[23730]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:07:26.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:26.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:26 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:27 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:27 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:27 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:28.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[28528]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:28.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.563 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[23730]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:28.563 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.563 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.564 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.821 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:28 smithi121 ceph-mon[24681]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:28.821 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.821 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:28.821 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[28528]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:30.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[23730]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:30 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:30 smithi121 ceph-mon[24681]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:30 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:30 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:30 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:30 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:32.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:07:31] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.43.0" 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[23730]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:31 smithi114 ceph-mon[28528]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:32.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:31 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:31 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:31 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:32.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:31 smithi121 ceph-mon[24681]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:34.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:34 smithi114 ceph-mon[28528]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:34.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:34 smithi114 ceph-mon[23730]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:34.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:34 smithi121 ceph-mon[24681]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:36 smithi114 ceph-mon[23730]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:37.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:36 smithi114 ceph-mon[28528]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:36 smithi121 ceph-mon[24681]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:37 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:38.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:37 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:37 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:38 smithi114 ceph-mon[23730]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:39.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:38 smithi114 ceph-mon[28528]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:38 smithi121 ceph-mon[24681]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:41.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:40 smithi114 ceph-mon[23730]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:41.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:40 smithi114 ceph-mon[28528]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:40 smithi121 ceph-mon[24681]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:42.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:07:41] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.43.0" 2024-09-13T22:07:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:42 smithi114 ceph-mon[23730]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:43.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:42 smithi114 ceph-mon[28528]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:42 smithi121 ceph-mon[24681]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:44.974 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:44 smithi114 ceph-mon[28528]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:44.975 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:44 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:44.975 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:44 smithi114 ceph-mon[23730]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:44.975 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:44 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:44 smithi121 ceph-mon[24681]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:44 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:07:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:46 smithi121 ceph-mon[24681]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:46 smithi114 ceph-mon[23730]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:47.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:46 smithi114 ceph-mon[28528]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:47.766 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:47 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:47.889 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:47 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:47.890 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:47 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:48 smithi121 ceph-mon[24681]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:48.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:48.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[28528]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[23730]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:07:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:49 smithi114 ceph-mon[23730]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:07:50.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:49 smithi114 ceph-mon[28528]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:07:50.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:49 smithi121 ceph-mon[24681]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:07:52.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:07:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:07:51] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-09-13T22:07:53.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:52 smithi114 ceph-mon[23730]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:53.068 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:52 smithi114 ceph-mon[28528]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:52 smithi121 ceph-mon[24681]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:55.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:54 smithi114 ceph-mon[23730]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:07:55.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:54 smithi114 ceph-mon[28528]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:07:55.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:54 smithi121 ceph-mon[24681]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:07:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:56 smithi114 ceph-mon[23730]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:57.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:56 smithi114 ceph-mon[28528]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:56 smithi121 ceph-mon[24681]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:07:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:57 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:58.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:57 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:57 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:07:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:58 smithi114 ceph-mon[23730]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:59.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:58 smithi114 ceph-mon[28528]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:07:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:58 smithi121 ceph-mon[24681]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:00.022 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:07:59 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:07:59 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:00.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:07:59 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:00 smithi121 ceph-mon[24681]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:01.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:00 smithi114 ceph-mon[23730]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:01.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:00 smithi114 ceph-mon[28528]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:02.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:08:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:08:01] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-09-13T22:08:03.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:02 smithi114 ceph-mon[23730]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:03.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:02 smithi114 ceph-mon[28528]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:02 smithi121 ceph-mon[24681]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:04.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:03 smithi114 ceph-mon[23730]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:04.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:03 smithi114 ceph-mon[28528]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:03 smithi121 ceph-mon[24681]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:07.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:06 smithi114 ceph-mon[23730]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:06 smithi114 ceph-mon[28528]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:06 smithi121 ceph-mon[24681]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:07.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:07 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:07 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:07 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:08.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:08 smithi121 ceph-mon[24681]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:08.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:08.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:08.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:08.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:08.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:08.925 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[23730]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[28528]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:10 smithi114 ceph-mon[23730]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:10 smithi114 ceph-mon[28528]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:10 smithi121 ceph-mon[24681]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:12.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:08:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:08:11] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-09-13T22:08:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:12 smithi114 ceph-mon[23730]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:13.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:12 smithi114 ceph-mon[28528]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:12 smithi121 ceph-mon[24681]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:14 smithi114 ceph-mon[23730]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:14 smithi114 ceph-mon[28528]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:14 smithi121 ceph-mon[24681]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[23730]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[28528]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:08:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:16 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:16 smithi121 ceph-mon[24681]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:16 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:16 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:16 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:08:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:16 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:08:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:16 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:18.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:17 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:18.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:17 smithi114 ceph-mon[28528]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:17 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:17 smithi114 ceph-mon[23730]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:17 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:17 smithi121 ceph-mon[24681]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:20.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:20 smithi114 ceph-mon[23730]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:20.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:20 smithi114 ceph-mon[28528]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:20.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:20 smithi121 ceph-mon[24681]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:22.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:08:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:08:21] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-09-13T22:08:23.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:22 smithi114 ceph-mon[23730]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:22 smithi114 ceph-mon[28528]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:22 smithi121 ceph-mon[24681]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:25.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:24 smithi114 ceph-mon[28528]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:25.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:24 smithi114 ceph-mon[23730]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:24 smithi121 ceph-mon[24681]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:27.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:26 smithi114 ceph-mon[23730]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:27.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:26 smithi114 ceph-mon[28528]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:26 smithi121 ceph-mon[24681]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:27.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:27 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:28.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:27 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:27 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:28.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:28 smithi121 ceph-mon[24681]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:28.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:28.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:28.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:28.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:28.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:28.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[23730]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[28528]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:29.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:29 smithi121 ceph-mon[24681]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:29 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:29 smithi114 ceph-mon[23730]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:29 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:30.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:29 smithi114 ceph-mon[28528]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:30.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:29 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:32.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:08:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:08:31] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.43.0" 2024-09-13T22:08:33.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:32 smithi114 ceph-mon[23730]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:33.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:32 smithi114 ceph-mon[28528]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:33.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:32 smithi121 ceph-mon[24681]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:35.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:34 smithi114 ceph-mon[23730]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:35.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:34 smithi114 ceph-mon[28528]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:34 smithi121 ceph-mon[24681]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:36 smithi114 ceph-mon[23730]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:37.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:36 smithi114 ceph-mon[28528]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:36 smithi121 ceph-mon[24681]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:37 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:38.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:37 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:37 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:39.036 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:38 smithi121 ceph-mon[24681]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:38 smithi114 ceph-mon[23730]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:39.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:38 smithi114 ceph-mon[28528]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:39.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:39 smithi121 ceph-mon[24681]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:40.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:39 smithi114 ceph-mon[28528]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:39 smithi114 ceph-mon[23730]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:42.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:08:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:08:41] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.43.0" 2024-09-13T22:08:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:42 smithi114 ceph-mon[23730]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:43.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:42 smithi114 ceph-mon[28528]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:42 smithi121 ceph-mon[24681]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:44 smithi114 ceph-mon[28528]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:44 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:44 smithi114 ceph-mon[23730]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:45.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:44 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:44 smithi121 ceph-mon[24681]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:44 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:08:47.049 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:46 smithi121 ceph-mon[24681]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:46 smithi114 ceph-mon[23730]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:47.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:46 smithi114 ceph-mon[28528]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:47.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:47 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:47.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:47 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:48.118 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:47 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[23730]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.994 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[28528]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:48.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:48.995 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:48 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:48 smithi121 ceph-mon[24681]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:49.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:49.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:49.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:49.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:48 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:08:49.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:49 smithi114 ceph-mon[23730]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:49.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:49 smithi114 ceph-mon[28528]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:49.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:49 smithi121 ceph-mon[24681]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:52.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:08:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:08:51] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.43.0" 2024-09-13T22:08:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:52 smithi114 ceph-mon[23730]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:53.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:52 smithi114 ceph-mon[28528]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:52 smithi121 ceph-mon[24681]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:55.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:54 smithi114 ceph-mon[23730]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:55.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:54 smithi114 ceph-mon[28528]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:55.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:54 smithi121 ceph-mon[24681]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:56 smithi114 ceph-mon[23730]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:57.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:56 smithi114 ceph-mon[28528]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:56 smithi121 ceph-mon[24681]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:08:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:57 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:58.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:57 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:57 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:08:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:58 smithi114 ceph-mon[23730]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:59.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:58 smithi114 ceph-mon[28528]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:08:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:58 smithi121 ceph-mon[24681]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:08:59 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:00.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:08:59 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:08:59 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:01.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:00 smithi114 ceph-mon[23730]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:01.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:00 smithi114 ceph-mon[28528]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:00 smithi121 ceph-mon[24681]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:02.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:01 smithi114 ceph-mon[23730]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:02.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:01 smithi114 ceph-mon[28528]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:02.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:09:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:09:01] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.43.0" 2024-09-13T22:09:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:01 smithi121 ceph-mon[24681]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:04.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:04 smithi114 ceph-mon[23730]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:04.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:04 smithi114 ceph-mon[28528]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:04.825 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:04 smithi121 ceph-mon[24681]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:06.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:06 smithi114 ceph-mon[23730]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:06.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:06 smithi114 ceph-mon[28528]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:07.094 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:06 smithi121 ceph-mon[24681]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:07 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:07 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:07 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[23730]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[28528]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:09.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:08 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:08 smithi121 ceph-mon[24681]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:09.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:08 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:10 smithi114 ceph-mon[23730]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:10 smithi114 ceph-mon[28528]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:10 smithi121 ceph-mon[24681]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:12.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:09:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:09:11] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.43.0" 2024-09-13T22:09:13.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:12 smithi114 ceph-mon[28528]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:12 smithi114 ceph-mon[23730]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:12 smithi121 ceph-mon[24681]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:13 smithi114 ceph-mon[28528]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:14.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:13 smithi114 ceph-mon[23730]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:13 smithi121 ceph-mon[24681]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:15.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:16.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:16 smithi114 ceph-mon[23730]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:16.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:16 smithi114 ceph-mon[28528]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:16 smithi121 ceph-mon[24681]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:18.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:09:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:09:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:18.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:18.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:09:18.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:09:18.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:17 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:17 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:17 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:09:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:17 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:09:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:17 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:19.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:18 smithi114 ceph-mon[28528]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:18 smithi114 ceph-mon[23730]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:18 smithi121 ceph-mon[24681]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:20 smithi114 ceph-mon[23730]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:21.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:20 smithi114 ceph-mon[28528]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:20 smithi121 ceph-mon[24681]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:22.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:09:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:09:21] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-09-13T22:09:22.990 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:22 smithi114 ceph-mon[23730]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:22.990 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:22 smithi114 ceph-mon[28528]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:22 smithi121 ceph-mon[24681]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:24 smithi114 ceph-mon[23730]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:25.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:24 smithi114 ceph-mon[28528]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:24 smithi121 ceph-mon[24681]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:26.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:26 smithi114 ceph-mon[23730]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:26 smithi114 ceph-mon[28528]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:26.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:26 smithi121 ceph-mon[24681]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:28.024 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:27 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:28.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:27 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:28.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:27 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[28528]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[23730]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:28 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.085 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:28 smithi121 ceph-mon[24681]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:29.085 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.085 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:29.085 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:28 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[23730]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[28528]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:30.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:29 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:29 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:29 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:29 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:29 smithi121 ceph-mon[24681]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:29 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:32.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:09:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:09:31] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.43.0" 2024-09-13T22:09:33.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:32 smithi114 ceph-mon[23730]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:33.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:32 smithi114 ceph-mon[28528]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:33.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:32 smithi121 ceph-mon[24681]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:35.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:34 smithi114 ceph-mon[23730]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:35.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:34 smithi114 ceph-mon[28528]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:34 smithi121 ceph-mon[24681]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:36.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:36 smithi114 ceph-mon[23730]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:36.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:36 smithi114 ceph-mon[28528]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:37.092 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:36 smithi121 ceph-mon[24681]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:37 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:38.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:37 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:37 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:38 smithi114 ceph-mon[23730]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:39.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:38 smithi114 ceph-mon[28528]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:38 smithi121 ceph-mon[24681]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:39 smithi114 ceph-mon[23730]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:39 smithi114 ceph-mon[28528]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:39 smithi121 ceph-mon[24681]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:42.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:09:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:09:41] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.43.0" 2024-09-13T22:09:43.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:42 smithi114 ceph-mon[23730]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:43.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:42 smithi114 ceph-mon[28528]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:42 smithi121 ceph-mon[24681]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:45.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:44 smithi114 ceph-mon[23730]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:45.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:44 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:44 smithi114 ceph-mon[28528]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:44 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:44 smithi121 ceph-mon[24681]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:44 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:09:46.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:46 smithi114 ceph-mon[23730]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:46.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:46 smithi114 ceph-mon[28528]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:46 smithi121 ceph-mon[24681]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:47 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:48.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:47 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:48.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:47 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:49.036 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:48 smithi121 ceph-mon[24681]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:49.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:48 smithi114 ceph-mon[23730]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:48 smithi114 ceph-mon[28528]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:50.073 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:49 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.073 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:49 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.073 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:49 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:49 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:49 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:49 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:49 smithi121 ceph-mon[24681]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[23730]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:09:50.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:49 smithi114 ceph-mon[28528]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:51.866 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:09:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:09:51] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.43.0" 2024-09-13T22:09:53.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:52 smithi114 ceph-mon[23730]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:53.121 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:52 smithi114 ceph-mon[28528]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:52 smithi121 ceph-mon[24681]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:54.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:54 smithi114 ceph-mon[23730]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:54.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:54 smithi114 ceph-mon[28528]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:55.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:54 smithi121 ceph-mon[24681]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:56.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:56 smithi114 ceph-mon[23730]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:56.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:56 smithi114 ceph-mon[28528]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:56 smithi121 ceph-mon[24681]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:09:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:57 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:58.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:57 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:57 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:09:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:58 smithi114 ceph-mon[23730]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:59.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:58 smithi114 ceph-mon[28528]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:09:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:58 smithi121 ceph-mon[24681]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:59 smithi114 ceph-mon[23730]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:00.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:09:59 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:00.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:59 smithi114 ceph-mon[28528]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:00.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:09:59 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:59 smithi121 ceph-mon[24681]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:09:59 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:01.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:00 smithi114 ceph-mon[23730]: overall HEALTH_OK 2024-09-13T22:10:01.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:00 smithi114 ceph-mon[28528]: overall HEALTH_OK 2024-09-13T22:10:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:00 smithi121 ceph-mon[24681]: overall HEALTH_OK 2024-09-13T22:10:02.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:01 smithi114 ceph-mon[23730]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:02.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:10:01] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.43.0" 2024-09-13T22:10:02.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:01 smithi114 ceph-mon[28528]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:01 smithi121 ceph-mon[24681]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:04.290 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:10:04.557 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:04 smithi114 ceph-mon[23730]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:04.557 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:04 smithi114 ceph-mon[28528]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:04.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:04 smithi121 ceph-mon[24681]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 17s ago 22m - - 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 17s ago 22m - - 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (9m) 17s ago 17m 25.5M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (8m) 17s ago 17m 56.8M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (2m) 17s ago 16m 46.8M - 3.5 e1d6a67b021e 93b7d16469e0 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283 running (12m) 17s ago 21m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 56081b4c7747 2024-09-13T22:10:06.289 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (5m) 17s ago 23m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (23m) 17s ago 23m 71.9M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (22m) 17s ago 22m 53.8M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (22m) 17s ago 22m 51.0M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (9m) 17s ago 18m 14.7M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (8m) 17s ago 17m 14.7M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (21m) 17s ago 21m 55.9M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (21m) 17s ago 21m 55.4M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (20m) 17s ago 20m 49.8M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (20m) 17s ago 20m 50.0M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (20m) 17s ago 20m 51.8M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (19m) 17s ago 19m 53.4M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (19m) 17s ago 19m 50.6M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:10:06.290 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (18m) 17s ago 18m 52.2M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:10:06.291 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (2m) 17s ago 18m 59.2M - 2.43.0 a07b618ecd1d eba628b49c89 2024-09-13T22:10:06.291 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (16m) 17s ago 16m 74.2M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:10:06.291 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (16m) 17s ago 16m 75.2M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:10:06.858 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T22:10:06.881 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:06 smithi114 ceph-mon[23730]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:06.882 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:06 smithi114 ceph-mon[28528]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:07.056 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:06 smithi121 ceph-mon[24681]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:07 smithi114 ceph-mon[28528]: from='client.25183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:10:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:07 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:07 smithi114 ceph-mon[23730]: from='client.25183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:10:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:07 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:07 smithi121 ceph-mon[24681]: from='client.25183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:10:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:07 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:08.902 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:10:08.902 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:10:08.902 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:10:08.902 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:10:08.902 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:10:08.903 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:10:09.062 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:08 smithi114 ceph-mon[23730]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:09.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:08 smithi114 ceph-mon[28528]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:08 smithi121 ceph-mon[24681]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:09.521 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph -s' 2024-09-13T22:10:09.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2772340861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:10:09.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2772340861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:09 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:10.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:09 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2772340861' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:10:10.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:09 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:10.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:09 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:10.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:09 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:10.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:09 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:10.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:09 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:10.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:09 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:10 smithi114 ceph-mon[23730]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:10 smithi114 ceph-mon[28528]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:10 smithi121 ceph-mon[24681]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:11.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:10:11] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.43.0" 2024-09-13T22:10:12.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:12 smithi114 ceph-mon[23730]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:12.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:12 smithi114 ceph-mon[28528]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:12.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:12 smithi121 ceph-mon[24681]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:12.971 INFO:teuthology.orchestra.run.smithi114.stdout: cluster: 2024-09-13T22:10:12.971 INFO:teuthology.orchestra.run.smithi114.stdout: id: 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: health: HEALTH_OK 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: services: 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: mon: 3 daemons, quorum a,c,b (age 22m) 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: mgr: y(active, since 3m), standbys: x 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 19m) 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: data: 2024-09-13T22:10:12.972 INFO:teuthology.orchestra.run.smithi114.stdout: pools: 6 pools, 161 pgs 2024-09-13T22:10:12.973 INFO:teuthology.orchestra.run.smithi114.stdout: objects: 209 objects, 585 KiB 2024-09-13T22:10:12.973 INFO:teuthology.orchestra.run.smithi114.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:12.973 INFO:teuthology.orchestra.run.smithi114.stdout: pgs: 161 active+clean 2024-09-13T22:10:12.973 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:10:12.973 INFO:teuthology.orchestra.run.smithi114.stdout: io: 2024-09-13T22:10:12.973 INFO:teuthology.orchestra.run.smithi114.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-09-13T22:10:12.973 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-09-13T22:10:13.583 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T22:10:13.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:13 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1433415773' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:10:13.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:13 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1433415773' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:10:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:13 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1433415773' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:10:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:14 smithi114 ceph-mon[23730]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:14 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:14 smithi114 ceph-mon[28528]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:15.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:14 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:14 smithi121 ceph-mon[24681]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:14 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:15.817 INFO:teuthology.orchestra.run.smithi114.stdout:HEALTH_OK 2024-09-13T22:10:16.403 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-13T22:10:16.967 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:16 smithi114 ceph-mon[23730]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:16.968 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:16 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2357860752' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:10:16.968 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:16 smithi114 ceph-mon[28528]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:16.968 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:16 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2357860752' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:10:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:16 smithi121 ceph-mon[24681]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:16 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2357860752' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:10:17.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:17.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:17.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[23730]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:17.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[23730]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[28528]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:17.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:17 smithi114 ceph-mon[28528]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:17 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:17 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:17 smithi121 ceph-mon[24681]: from='mgr.25093 172.21.15.114:0/1435090996' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:18.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:17 smithi121 ceph-mon[24681]: from='mgr.25093 ' entity='mgr.y' 2024-09-13T22:10:18.419 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:10:18.976 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr fail' 2024-09-13T22:10:18.999 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:18 smithi114 ceph-mon[23730]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:18.999 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:18 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3325280134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:10:18.999 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:18 smithi114 ceph-mon[28528]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:18.999 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:18 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3325280134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:10:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:18 smithi121 ceph-mon[24681]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:18 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3325280134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:10:21.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:20 smithi114 ceph-mon[23730]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:21.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:20 smithi114 ceph-mon[28528]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:20 smithi121 ceph-mon[24681]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:21.978 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/633431687' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:10:21.978 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-mon[23730]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T22:10:21.978 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-mon[23730]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/633431687' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-mon[28528]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-mon[28528]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:10:21] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.43.0" 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:21.760+0000 7fe3ac2c4640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ignoring --setuser ceph since I am not root 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ignoring --setgroup ceph since I am not root 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-mgr[70608]: -- 172.21.15.114:0/945392252 <== mon.1 v2:172.21.15.114:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563658bdb0e0 con 0x563658bb8800 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:21.920+0000 7f2152a08040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:10:21.979 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:21.977+0000 7f2152a08040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:10:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:21 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/633431687' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-13T22:10:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:21 smithi121 ceph-mon[24681]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T22:10:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:21 smithi121 ceph-mon[24681]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:10:22.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:10:21] ENGINE Bus STOPPING 2024-09-13T22:10:22.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:10:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:10:22.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:10:21] ENGINE Bus STOPPED 2024-09-13T22:10:22.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:22.128+0000 7f2152a08040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:10:22.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:22.194+0000 7f2152a08040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:10:22.590 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:22.344+0000 7f2152a08040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:10:22.590 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:22.488+0000 7f2152a08040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:10:22.592 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 180' 2024-09-13T22:10:22.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:10:22] ENGINE Bus STARTING 2024-09-13T22:10:22.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:10:22] ENGINE Serving on http://:::9283 2024-09-13T22:10:22.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:10:22] ENGINE Bus STARTED 2024-09-13T22:10:22.885 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:22.815+0000 7f2152a08040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:10:22.885 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:22.884+0000 7f2152a08040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:10:22.885 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/633431687' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:10:22.885 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: mgrmap e31: x(active, starting, since 0.851474s) 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:10:22.886 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: Manager daemon x is now available 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:10:22.887 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/633431687' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: mgrmap e31: x(active, starting, since 0.851474s) 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:10:22.888 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: Manager daemon x is now available 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:10:22.889 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:10:22.890 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:22.890 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:10:22.890 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:10:22.890 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:10:22.890 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:22 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:10:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/633431687' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: mgrmap e31: x(active, starting, since 0.851474s) 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:10:23.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: Manager daemon x is now available 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:10:23.123 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:10:23.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:10:23.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:22 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T22:10:23.188 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:23.079+0000 7f2152a08040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:10:23.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: mgrmap e32: x(active, since 2s) 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:23.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: mgrmap e32: x(active, since 2s) 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:23.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:23 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: mgrmap e32: x(active, since 2s) 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:23.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:23 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:24.098 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:24.052+0000 7f2152a08040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:10:24.352 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:24.125+0000 7f2152a08040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:10:24.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:10:24.223+0000 7fb7ee4df640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:22:10:24] ENGINE Error in HTTPServer.serve 2024-09-13T22:10:24.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self._connections.run(self.expiration_interval) 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self._run(expiration_interval) 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: s = self.context.wrap_socket( 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:10:24.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.sslsocket_class._create( 2024-09-13T22:10:24.623 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:10:24.623 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self.do_handshake() 2024-09-13T22:10:24.623 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:10:24.623 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self._sslobj.do_handshake() 2024-09-13T22:10:24.623 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:10:24.623 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:10:25.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:24.954+0000 7f2152a08040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:10:25.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.081+0000 7f2152a08040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:10:25.235 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.233+0000 7f2152a08040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:10:25.522 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.297+0000 7f2152a08040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.450+0000 7f2152a08040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.521+0000 7f2152a08040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: [13/Sep/2024:22:10:24] ENGINE Bus STARTING 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:10:25.523 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: [13/Sep/2024:22:10:24] ENGINE Serving on https://172.21.15.121:7150 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: [13/Sep/2024:22:10:24] ENGINE Error in HTTPServer.serve 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:10:25.524 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.do_handshake() 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: [13/Sep/2024:22:10:24] ENGINE Serving on http://172.21.15.121:8765 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: [13/Sep/2024:22:10:24] ENGINE Bus STARTED 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.525 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.526 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: [13/Sep/2024:22:10:24] ENGINE Bus STARTING 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: [13/Sep/2024:22:10:24] ENGINE Serving on https://172.21.15.121:7150 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: [13/Sep/2024:22:10:24] ENGINE Error in HTTPServer.serve 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:10:25.527 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.do_handshake() 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: [13/Sep/2024:22:10:24] ENGINE Serving on http://172.21.15.121:8765 2024-09-13T22:10:25.528 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: [13/Sep/2024:22:10:24] ENGINE Bus STARTED 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.529 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.530 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:25.530 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:25.530 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:25.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:25.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:25.530 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:25.598 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:10:25.598 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:10:25.598 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:25.598 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: [13/Sep/2024:22:10:24] ENGINE Bus STARTING 2024-09-13T22:10:25.598 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:10:25.598 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: [13/Sep/2024:22:10:24] ENGINE Serving on https://172.21.15.121:7150 2024-09-13T22:10:25.598 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: [13/Sep/2024:22:10:24] ENGINE Error in HTTPServer.serve 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._run(expiration_interval) 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:10:25.599 INFO:journalctl@ceph.mon.b.smithi121.stdout: s = self.context.wrap_socket( 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.sslsocket_class._create( 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.do_handshake() 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._sslobj.do_handshake() 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: [13/Sep/2024:22:10:24] ENGINE Serving on http://172.21.15.121:8765 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: [13/Sep/2024:22:10:24] ENGINE Bus STARTED 2024-09-13T22:10:25.600 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.601 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.602 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.602 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.602 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:25.602 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:25.602 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:25.602 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:25 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:25.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.582+0000 7f2152a08040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:10:25.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.767+0000 7f2152a08040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:10:26.045 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.852+0000 7f2152a08040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:10:26.046 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:25.930+0000 7f2152a08040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:10:26.046 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:26.018+0000 7f2152a08040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:10:26.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:26.085+0000 7f2152a08040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:10:26.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:26.147+0000 7f2152a08040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:10:26.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:10:26.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:10:26.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:10:26.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:10:26.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:10:26.523 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: mgrmap e33: x(active, since 4s) 2024-09-13T22:10:26.523 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.523 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.523 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.523 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.523 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.523 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:26 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.732 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:10:26.732 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:10:26.732 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:10:26.732 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:10:26.732 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:10:26.732 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: mgrmap e33: x(active, since 4s) 2024-09-13T22:10:26.732 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: mgrmap e33: x(active, since 4s) 2024-09-13T22:10:26.733 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.734 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.734 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.734 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.734 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:26.734 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:26.985+0000 7f2152a08040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:10:27.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:10:26] ENGINE Bus STARTING 2024-09-13T22:10:27.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: CherryPy Checker: 2024-09-13T22:10:27.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: The Application mounted at '' has an empty config. 2024-09-13T22:10:27.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:10:27.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:10:27] ENGINE Serving on http://:::9283 2024-09-13T22:10:27.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:10:27] ENGINE Bus STARTED 2024-09-13T22:10:27.561 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: Standby manager daemon y started 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: Standby manager daemon y started 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:27 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: Standby manager daemon y started 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:27 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.475 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[23730]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[23730]: mgrmap e34: x(active, since 6s), standbys: y 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[28528]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.476 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[28528]: mgrmap e34: x(active, since 6s), standbys: y 2024-09-13T22:10:28.477 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:28 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:10:28.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:28 smithi121 ceph-mon[24681]: from='client.25168 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:28.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:28 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:28 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:28 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:28.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:28 smithi121 ceph-mon[24681]: mgrmap e34: x(active, since 6s), standbys: y 2024-09-13T22:10:28.681 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:28 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:10:29.553 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:29 smithi121 ceph-mon[24681]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:29 smithi114 ceph-mon[23730]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:29.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:29 smithi114 ceph-mon[28528]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:30.779 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.779 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.779 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.779 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[23730]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:10:30.780 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.780 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.780 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.780 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:30 smithi114 ceph-mon[28528]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:10:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-mon[24681]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:10:31.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:10:30.974+0000 7fb8227c4640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:31.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Creating ceph-iscsi config... 2024-09-13T22:10:31.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return _run_code(code, main_globals, None, 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: exec(code, run_globals) 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:31.372 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:31.373 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:31.373 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:31.373 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:31.373 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.373 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.373 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:10:31.373 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.wait_async( 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return future.result(timeout) 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return self.__get_result() 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: raise self._exception 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: out, err, code = await self._run_cephadm( 2024-09-13T22:10:31.374 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: raise OrchestratorError( 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Creating ceph-iscsi config... 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: Traceback (most recent call last): 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: return _run_code(code, main_globals, None, 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: exec(code, run_globals) 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:31.375 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:31.376 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:31.376 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:31.376 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:31.376 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:31.376 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:31.376 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.376 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:31 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:10:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:10:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:31.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:31.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:31 smithi121 ceph-mon[24681]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:10:31.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:31 smithi121 ceph-mon[24681]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:31 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:10:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:10:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:10:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:10:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:10:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:10:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:10:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:10:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:10:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:10:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:31 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:10:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:10:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:10:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:10:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:31 smithi114 ceph-mon[23730]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:31 smithi114 ceph-mon[23730]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:10:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:10:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:10:32.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:31 smithi114 ceph-mon[28528]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:10:32.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:31 smithi114 ceph-mon[28528]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:10:32.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:32 smithi121 ceph-mon[24681]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:10:33.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:32 smithi114 ceph-mon[28528]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:10:33.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:32 smithi114 ceph-mon[23730]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:10:34.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.078 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:34.371 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 systemd[1]: Stopping Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:10:34.851 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T22:10:34.851 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[62252]: ts=2024-09-13T22:10:34.482Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T22:10:34.852 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 podman[66714]: 2024-09-13 22:10:34.492612537 +0000 UTC m=+0.126948714 container died eba628b49c89c89334d50f1b2396929a2e145618c486d6c3c750fd2eafa4bb0a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:10:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:34 smithi121 ceph-mon[24681]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:10:35.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 podman[66714]: 2024-09-13 22:10:34.965969061 +0000 UTC m=+0.600305224 container cleanup eba628b49c89c89334d50f1b2396929a2e145618c486d6c3c750fd2eafa4bb0a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:10:35.121 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:34 smithi121 bash[66714]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T22:10:35.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:34 smithi114 ceph-mon[23730]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:10:35.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:34 smithi114 ceph-mon[28528]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:10:35.594 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 podman[66726]: 2024-09-13 22:10:35.225421305 +0000 UTC m=+0.734471723 container remove eba628b49c89c89334d50f1b2396929a2e145618c486d6c3c750fd2eafa4bb0a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:10:35.595 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T22:10:35.595 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 systemd[1]: Stopped Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:10:35.595 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.657s CPU time. 2024-09-13T22:10:35.595 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 systemd[1]: Starting Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:10:35.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 podman[66819]: 2024-09-13 22:10:35.497778159 +0000 UTC m=+0.016600475 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T22:10:35.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 podman[66819]: 2024-09-13 22:10:35.66229886 +0000 UTC m=+0.181121174 container create c0aa16eafc042269120313b482e0f3d5e52c613f394a4f93272ce2970dea5d6a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 podman[66819]: 2024-09-13 22:10:35.920606531 +0000 UTC m=+0.439428861 container init c0aa16eafc042269120313b482e0f3d5e52c613f394a4f93272ce2970dea5d6a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 podman[66819]: 2024-09-13 22:10:35.923568745 +0000 UTC m=+0.442391066 container start c0aa16eafc042269120313b482e0f3d5e52c613f394a4f93272ce2970dea5d6a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.947Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.947Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.947Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi121 (none))" 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.947Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.947Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.949Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T22:10:36.372 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.949Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.950Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.950Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.952Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.953Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.331584ms 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.953Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.954Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.972Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.982Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.984Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.985Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.985Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.603µs wal_replay_duration=31.395244ms wbl_replay_duration=116ns total_replay_duration=32.772481ms 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.988Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.988Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T22:10:36.373 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:35.988Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T22:10:36.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:36 smithi121 bash[66819]: c0aa16eafc042269120313b482e0f3d5e52c613f394a4f93272ce2970dea5d6a 2024-09-13T22:10:36.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:36 smithi121 systemd[1]: Started Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:10:36.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:36.011Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.287363ms db_storage=730ns remote_storage=1.45µs web_handler=470ns query_engine=555ns scrape=347.828µs scrape_sd=94.836µs notify=9.715µs notify_sd=9.154µs rules=22.454328ms tracing=6.869µs 2024-09-13T22:10:36.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:36.011Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T22:10:36.374 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:10:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:10:36.011Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T22:10:37.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:37.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:10:37.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:10:37.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:10:37.286 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:37.287 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:37 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:10:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:37 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:10:38.146 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:38 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:38.147 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:38 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:38.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:38 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:39.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:39.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:39 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:10:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:10:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:10:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:10:39.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:39 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:40.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:10:40.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[23730]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:10:40.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3866399720' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:10:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2758769876' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/1470807499"}]: dispatch 2024-09-13T22:10:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:10:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[28528]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:10:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3866399720' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:10:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:40 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2758769876' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/1470807499"}]: dispatch 2024-09-13T22:10:40.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:40 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-13T22:10:40.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:40 smithi121 ceph-mon[24681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-13T22:10:40.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:40 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3866399720' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:10:40.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:40 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2758769876' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/1470807499"}]: dispatch 2024-09-13T22:10:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[23730]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2758769876' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/1470807499"}]': finished 2024-09-13T22:10:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[23730]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T22:10:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2959445414' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/1470807499"}]: dispatch 2024-09-13T22:10:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[28528]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:41.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2758769876' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/1470807499"}]': finished 2024-09-13T22:10:41.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[28528]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T22:10:41.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:41 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2959445414' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/1470807499"}]: dispatch 2024-09-13T22:10:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:41 smithi121 ceph-mon[24681]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:10:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:41 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2758769876' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6801/1470807499"}]': finished 2024-09-13T22:10:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:41 smithi121 ceph-mon[24681]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T22:10:41.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:41 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2959445414' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/1470807499"}]: dispatch 2024-09-13T22:10:42.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:42 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2959445414' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/1470807499"}]': finished 2024-09-13T22:10:42.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:42 smithi114 ceph-mon[28528]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T22:10:42.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:42 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2001814227' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4180630961"}]: dispatch 2024-09-13T22:10:42.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:42 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2959445414' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/1470807499"}]': finished 2024-09-13T22:10:42.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:42 smithi114 ceph-mon[23730]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T22:10:42.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:42 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2001814227' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4180630961"}]: dispatch 2024-09-13T22:10:42.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:42 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2959445414' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:6800/1470807499"}]': finished 2024-09-13T22:10:42.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:42 smithi121 ceph-mon[24681]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T22:10:42.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:42 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2001814227' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4180630961"}]: dispatch 2024-09-13T22:10:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:43 smithi121 ceph-mon[24681]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:43 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2001814227' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4180630961"}]': finished 2024-09-13T22:10:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:43 smithi121 ceph-mon[24681]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T22:10:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:43 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1609730798' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]: dispatch 2024-09-13T22:10:43.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:43 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]: dispatch 2024-09-13T22:10:43.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[23730]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:43.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2001814227' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4180630961"}]': finished 2024-09-13T22:10:43.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[23730]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T22:10:43.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1609730798' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]: dispatch 2024-09-13T22:10:43.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]: dispatch 2024-09-13T22:10:43.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[28528]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:10:43.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2001814227' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/4180630961"}]': finished 2024-09-13T22:10:43.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[28528]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T22:10:43.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1609730798' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]: dispatch 2024-09-13T22:10:43.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:43 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]: dispatch 2024-09-13T22:10:43.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:10:43] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.43.0" 2024-09-13T22:10:44.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:44 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]': finished 2024-09-13T22:10:44.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:44 smithi121 ceph-mon[24681]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T22:10:44.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:44 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/282880873' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]: dispatch 2024-09-13T22:10:44.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:44 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]: dispatch 2024-09-13T22:10:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]': finished 2024-09-13T22:10:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[28528]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T22:10:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/282880873' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]: dispatch 2024-09-13T22:10:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]: dispatch 2024-09-13T22:10:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2226299949"}]': finished 2024-09-13T22:10:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[23730]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T22:10:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/282880873' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]: dispatch 2024-09-13T22:10:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:44 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]: dispatch 2024-09-13T22:10:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:45 smithi121 ceph-mon[24681]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:45 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]': finished 2024-09-13T22:10:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:45 smithi121 ceph-mon[24681]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T22:10:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:45 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3699364205' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]: dispatch 2024-09-13T22:10:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:45 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]: dispatch 2024-09-13T22:10:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[23730]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]': finished 2024-09-13T22:10:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[23730]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T22:10:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3699364205' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]: dispatch 2024-09-13T22:10:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]: dispatch 2024-09-13T22:10:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[28528]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/1246958799"}]': finished 2024-09-13T22:10:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[28528]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T22:10:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3699364205' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]: dispatch 2024-09-13T22:10:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:45 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]: dispatch 2024-09-13T22:10:46.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:46 smithi121 ceph-mon[24681]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]': finished 2024-09-13T22:10:46.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:46 smithi121 ceph-mon[24681]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T22:10:46.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:46 smithi114 ceph-mon[23730]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]': finished 2024-09-13T22:10:46.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:46 smithi114 ceph-mon[23730]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T22:10:46.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:46 smithi114 ceph-mon[28528]: from='client.? ' entity='client.iscsi.foo.smithi114.lemsof' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.114:0/2319984514"}]': finished 2024-09-13T22:10:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:46 smithi114 ceph-mon[28528]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T22:10:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:47 smithi121 ceph-mon[24681]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:47.425 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:47 smithi114 ceph-mon[23730]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:47.426 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:47 smithi114 ceph-mon[28528]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:10:49.163 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:49 smithi121 ceph-mon[24681]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 1 op/s 2024-09-13T22:10:49.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:49 smithi114 ceph-mon[28528]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 1 op/s 2024-09-13T22:10:49.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:49 smithi114 ceph-mon[23730]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 1 op/s 2024-09-13T22:10:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:50 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:50 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:50 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:50 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:50.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:50 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:51.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:51 smithi121 ceph-mon[24681]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 763 B/s rd, 0 op/s 2024-09-13T22:10:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:51 smithi114 ceph-mon[23730]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 763 B/s rd, 0 op/s 2024-09-13T22:10:51.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:51 smithi114 ceph-mon[28528]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 763 B/s rd, 0 op/s 2024-09-13T22:10:53.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T22:10:53.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:53.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.309 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.310 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T22:10:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:10:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:53.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:10:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:10:53] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.43.0" 2024-09-13T22:10:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[23730]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:10:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[28528]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:10:55.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:55 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:55 smithi121 ceph-mon[24681]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:10:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:55 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:55 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:55 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:10:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:56 smithi114 ceph-mon[23730]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-09-13T22:10:57.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:56 smithi114 ceph-mon[28528]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-09-13T22:10:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:56 smithi121 ceph-mon[24681]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-09-13T22:10:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:58 smithi114 ceph-mon[23730]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:59.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:58 smithi114 ceph-mon[28528]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:10:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:58 smithi121 ceph-mon[24681]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:10:59 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:00.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:10:59 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:00.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:10:59 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:00 smithi121 ceph-mon[24681]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:01.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:00 smithi114 ceph-mon[23730]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:00 smithi114 ceph-mon[28528]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:02 smithi121 ceph-mon[24681]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:03.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:02 smithi114 ceph-mon[23730]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:03.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:02 smithi114 ceph-mon[28528]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:03.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:11:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:11:03] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.43.0" 2024-09-13T22:11:05.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:04 smithi121 ceph-mon[24681]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:04 smithi114 ceph-mon[23730]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:05.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:04 smithi114 ceph-mon[28528]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:06 smithi114 ceph-mon[23730]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:06 smithi114 ceph-mon[28528]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:06 smithi121 ceph-mon[24681]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:08.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:07 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:07 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:07 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:09.035 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:08 smithi121 ceph-mon[24681]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:08 smithi114 ceph-mon[23730]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:09.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:08 smithi114 ceph-mon[28528]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:09 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:10.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:09 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:10.117 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:09 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[23730]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[28528]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:10 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:10 smithi121 ceph-mon[24681]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:10 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:10 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:10 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[23730]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:13.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[28528]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:13.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:13 smithi121 ceph-mon[24681]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:13.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:13.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:11:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:11:13] "GET /metrics HTTP/1.1" 200 37926 "" "Prometheus/2.43.0" 2024-09-13T22:11:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:15 smithi114 ceph-mon[23730]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:15.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:15 smithi114 ceph-mon[28528]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:15 smithi121 ceph-mon[24681]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:16 smithi114 ceph-mon[23730]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:17.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:16 smithi114 ceph-mon[28528]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:16 smithi121 ceph-mon[24681]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:18 smithi114 ceph-mon[23730]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:18 smithi114 ceph-mon[28528]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:18 smithi121 ceph-mon[24681]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:19 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:20.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:19 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:20.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:19 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:20 smithi121 ceph-mon[24681]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:20 smithi114 ceph-mon[23730]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:21.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:20 smithi114 ceph-mon[28528]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:22 smithi121 ceph-mon[24681]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:23.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:22 smithi114 ceph-mon[23730]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:23.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:23.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:22 smithi114 ceph-mon[28528]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:23.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:23.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:11:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:11:23] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.43.0" 2024-09-13T22:11:25.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:24 smithi121 ceph-mon[24681]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:25.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:24 smithi114 ceph-mon[23730]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:24 smithi114 ceph-mon[28528]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:27.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:26 smithi114 ceph-mon[23730]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:26 smithi114 ceph-mon[28528]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:26 smithi121 ceph-mon[24681]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:28.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:28 smithi121 ceph-mon[24681]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:28 smithi114 ceph-mon[23730]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:28 smithi114 ceph-mon[28528]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:30.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:29 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:30.111 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:29 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:30.111 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:29 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:30 smithi121 ceph-mon[24681]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:30 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:30 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:30 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[23730]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[28528]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:31.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:30 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[28528]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:33.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[23730]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:11:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:11:33] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.43.0" 2024-09-13T22:11:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:33 smithi121 ceph-mon[24681]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:35.673 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:35 smithi121 ceph-mon[24681]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:35.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:35 smithi114 ceph-mon[23730]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:35.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:35 smithi114 ceph-mon[28528]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:36 smithi114 ceph-mon[23730]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:37.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:36 smithi114 ceph-mon[28528]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:36 smithi121 ceph-mon[24681]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:38.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:38 smithi114 ceph-mon[23730]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:38 smithi114 ceph-mon[28528]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:38 smithi121 ceph-mon[24681]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:11:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:11:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:11:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:11:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:39 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:11:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:11:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:39 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:39 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:41.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:40 smithi114 ceph-mon[23730]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:41.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:40 smithi114 ceph-mon[28528]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:40 smithi121 ceph-mon[24681]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:42 smithi121 ceph-mon[24681]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:43.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:42 smithi114 ceph-mon[23730]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:43.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:42 smithi114 ceph-mon[28528]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:43.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:11:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:11:43] "GET /metrics HTTP/1.1" 200 37926 "" "Prometheus/2.43.0" 2024-09-13T22:11:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:44 smithi121 ceph-mon[24681]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:45.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:44 smithi114 ceph-mon[23730]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:45.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:44 smithi114 ceph-mon[28528]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:46 smithi114 ceph-mon[23730]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:47.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:46 smithi114 ceph-mon[28528]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:46 smithi121 ceph-mon[24681]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:49.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:48 smithi114 ceph-mon[23730]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:49.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:48 smithi114 ceph-mon[28528]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:49.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:48 smithi121 ceph-mon[24681]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:50.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:49 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:50.013 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:49 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:50.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:49 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:11:51.025 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:50 smithi114 ceph-mon[23730]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:51.025 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:50 smithi114 ceph-mon[28528]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:50 smithi121 ceph-mon[24681]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:51 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:51 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:51 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:51 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:51 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:51 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:51 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:51 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:52.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:51 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[23730]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:53.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[28528]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:52 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:52 smithi121 ceph-mon[24681]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:52 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:11:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:52 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:52 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:52 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:11:53.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:11:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:11:53] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-13T22:11:55.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:54 smithi114 ceph-mon[23730]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:55.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:54 smithi114 ceph-mon[28528]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:54 smithi121 ceph-mon[24681]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:11:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:56 smithi114 ceph-mon[23730]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:57.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:56 smithi114 ceph-mon[28528]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:56 smithi121 ceph-mon[24681]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:58 smithi121 ceph-mon[24681]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:59.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:58 smithi114 ceph-mon[23730]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:11:59.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:58 smithi114 ceph-mon[28528]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:11:59 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:00.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:11:59 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:00.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:11:59 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:00 smithi121 ceph-mon[24681]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:01.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:00 smithi114 ceph-mon[23730]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:01.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:00 smithi114 ceph-mon[28528]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:02.865 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:02 smithi114 ceph-mon[23730]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:02 smithi121 ceph-mon[24681]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:03.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:02 smithi114 ceph-mon[28528]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:03.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:12:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:12:03] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-13T22:12:05.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:04 smithi121 ceph-mon[24681]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:04 smithi114 ceph-mon[23730]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:05.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:04 smithi114 ceph-mon[28528]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:06 smithi114 ceph-mon[23730]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:06 smithi114 ceph-mon[28528]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:06 smithi121 ceph-mon[24681]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:07 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:07 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:07 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:08 smithi114 ceph-mon[23730]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:09.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:08 smithi114 ceph-mon[28528]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:08 smithi121 ceph-mon[24681]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:09.802 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:09 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:09 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:10.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:09 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:10.860 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:10 smithi114 ceph-mon[23730]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:10.861 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:10 smithi114 ceph-mon[28528]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:11.089 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:10 smithi121 ceph-mon[24681]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:12.108 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:12 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:12 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:12 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.109 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:12 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.109 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:12 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.109 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:12 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:12 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:12 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:12.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:12 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:13 smithi121 ceph-mon[24681]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.429 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[23730]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:13.429 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.429 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.429 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.430 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[28528]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:13.430 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.430 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.430 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:13.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:12:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:12:13] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.43.0" 2024-09-13T22:12:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:15 smithi121 ceph-mon[24681]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:15 smithi114 ceph-mon[23730]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:15.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:15 smithi114 ceph-mon[28528]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:16 smithi114 ceph-mon[23730]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:17.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:16 smithi114 ceph-mon[28528]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:16 smithi121 ceph-mon[24681]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:19.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:18 smithi121 ceph-mon[24681]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:18 smithi114 ceph-mon[23730]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:19.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:18 smithi114 ceph-mon[28528]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:19 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:20.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:19 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:20.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:19 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:20 smithi121 ceph-mon[24681]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:21.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:20 smithi114 ceph-mon[28528]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:20 smithi114 ceph-mon[23730]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:23.127 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:22 smithi114 ceph-mon[23730]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:23.127 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:23.127 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:22 smithi114 ceph-mon[28528]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:23.127 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:22 smithi121 ceph-mon[24681]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:23.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:12:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:12:23] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-09-13T22:12:25.312 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:24 smithi114 ceph-mon[23730]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:25.313 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:24 smithi114 ceph-mon[28528]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:25.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:24 smithi121 ceph-mon[24681]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:27.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:26 smithi114 ceph-mon[23730]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:26 smithi114 ceph-mon[28528]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:26 smithi121 ceph-mon[24681]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:29.029 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:28 smithi121 ceph-mon[24681]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:29.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:28 smithi114 ceph-mon[23730]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:29.259 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:28 smithi114 ceph-mon[28528]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:29 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:29 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:30.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:29 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:30.898 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:30 smithi121 ceph-mon[24681]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:31.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:30 smithi114 ceph-mon[23730]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:31.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:30 smithi114 ceph-mon[28528]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:32.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:32 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:32 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:32 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:32 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:32 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:32 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:32 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:32 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:32.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:32 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:33 smithi121 ceph-mon[24681]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[23730]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:33.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[28528]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:33.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:33.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:12:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:12:33] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-09-13T22:12:35.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:35 smithi114 ceph-mon[28528]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:35 smithi114 ceph-mon[23730]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:35.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:35 smithi121 ceph-mon[24681]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:36 smithi114 ceph-mon[23730]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:37.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:36 smithi114 ceph-mon[28528]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:36 smithi121 ceph-mon[24681]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:38.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:38.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:38 smithi114 ceph-mon[23730]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:39.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:38 smithi114 ceph-mon[28528]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:38 smithi121 ceph-mon[24681]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-13T22:12:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:12:40.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:12:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:12:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:12:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:39 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:39 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:12:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:12:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:39 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:41.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:40 smithi114 ceph-mon[23730]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:41.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:40 smithi114 ceph-mon[28528]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:40 smithi121 ceph-mon[24681]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:42.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:42 smithi121 ceph-mon[24681]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:43.262 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:42 smithi114 ceph-mon[23730]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:43.262 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:42 smithi114 ceph-mon[28528]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:43.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:12:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:12:43] "GET /metrics HTTP/1.1" 200 37933 "" "Prometheus/2.43.0" 2024-09-13T22:12:45.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:44 smithi121 ceph-mon[24681]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:45.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:44 smithi114 ceph-mon[23730]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:45.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:44 smithi114 ceph-mon[28528]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:46 smithi114 ceph-mon[23730]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:47.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:46 smithi114 ceph-mon[28528]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:46 smithi121 ceph-mon[24681]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:48.906 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:48 smithi121 ceph-mon[24681]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:49.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:48 smithi114 ceph-mon[23730]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:49.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:48 smithi114 ceph-mon[28528]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:50.011 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:49 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:49 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:50.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:49 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:12:51.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:50 smithi121 ceph-mon[24681]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:50 smithi114 ceph-mon[23730]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:51.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:50 smithi114 ceph-mon[28528]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:52.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:52 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:52 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:52 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:52 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:52 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:52 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:52 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:52 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:52.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:52 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[23730]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:53.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:53.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[28528]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:53 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:12:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:12:53] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-09-13T22:12:53.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:53 smithi121 ceph-mon[24681]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:53.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:53 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:12:53.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:53.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:53 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:12:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:55 smithi114 ceph-mon[23730]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:55.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:55 smithi114 ceph-mon[28528]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:55 smithi121 ceph-mon[24681]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:12:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:56 smithi114 ceph-mon[23730]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:56 smithi114 ceph-mon[28528]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:57.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:56 smithi121 ceph-mon[24681]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:58 smithi121 ceph-mon[24681]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:59.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:58 smithi114 ceph-mon[23730]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:12:59.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:58 smithi114 ceph-mon[28528]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:00.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:12:59 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:00.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:12:59 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:00.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:12:59 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:01.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:00 smithi114 ceph-mon[23730]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:01.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:00 smithi114 ceph-mon[28528]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:00 smithi121 ceph-mon[24681]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:03.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:02 smithi114 ceph-mon[23730]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:03.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:02 smithi114 ceph-mon[28528]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:03.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:02 smithi121 ceph-mon[24681]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:03.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:13:03] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-09-13T22:13:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:04 smithi114 ceph-mon[23730]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:05.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:04 smithi114 ceph-mon[28528]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:04 smithi121 ceph-mon[24681]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:07.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:06 smithi114 ceph-mon[23730]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:06 smithi114 ceph-mon[28528]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:06 smithi121 ceph-mon[24681]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:07 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:07 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:07 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:09.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:08 smithi114 ceph-mon[28528]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:08 smithi114 ceph-mon[23730]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:08 smithi121 ceph-mon[24681]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:09 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:10.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:09 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:09 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:11.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:10 smithi121 ceph-mon[24681]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:10 smithi114 ceph-mon[23730]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:11.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:10 smithi114 ceph-mon[28528]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:12.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:12 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:12 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:12 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:12 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:12 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:12 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:12 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:12 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:12.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:12 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[23730]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:13.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[28528]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:13 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:13:13] "GET /metrics HTTP/1.1" 200 37934 "" "Prometheus/2.43.0" 2024-09-13T22:13:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:13 smithi121 ceph-mon[24681]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:13 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:15.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:15 smithi114 ceph-mon[23730]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:15.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:15 smithi114 ceph-mon[28528]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:15.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:15 smithi121 ceph-mon[24681]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:17.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:16 smithi114 ceph-mon[23730]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:17.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:16 smithi114 ceph-mon[28528]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:16 smithi121 ceph-mon[24681]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:18 smithi114 ceph-mon[23730]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:18 smithi114 ceph-mon[28528]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:19.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:18 smithi121 ceph-mon[24681]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:20.172 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:19 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:20.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:19 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:20.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:19 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:20 smithi114 ceph-mon[23730]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:21.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:20 smithi114 ceph-mon[28528]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:20 smithi121 ceph-mon[24681]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:23.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:22 smithi114 ceph-mon[28528]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:23.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:22 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:23.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:22 smithi114 ceph-mon[23730]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:23.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:22 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:22 smithi121 ceph-mon[24681]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:22 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:23.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:13:23] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-09-13T22:13:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:24 smithi114 ceph-mon[23730]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:25.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:24 smithi114 ceph-mon[28528]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:25.153 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:13:25.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:24 smithi121 ceph-mon[24681]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:27.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:26 smithi114 ceph-mon[23730]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:27.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:26 smithi114 ceph-mon[28528]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:27.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:26 smithi121 ceph-mon[24681]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:27.285 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:13:27.285 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 14s ago 26m - - 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 15s ago 25m - - 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (12m) 14s ago 20m 25.5M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (11m) 15s ago 20m 56.4M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (2m) 14s ago 19m 52.8M - 3.5 e1d6a67b021e 32b02f8593e2 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283 running (15m) 15s ago 25m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 56081b4c7747 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (8m) 14s ago 26m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (26m) 14s ago 27m 77.8M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (25m) 15s ago 25m 61.7M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (25m) 14s ago 25m 56.1M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (12m) 14s ago 21m 15.4M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (12m) 15s ago 21m 15.1M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (25m) 14s ago 25m 56.4M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (24m) 14s ago 24m 56.2M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:13:27.286 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (24m) 14s ago 24m 49.2M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (23m) 14s ago 23m 50.5M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (23m) 15s ago 23m 52.2M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (23m) 15s ago 23m 53.9M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (22m) 15s ago 22m 51.0M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (22m) 15s ago 22m 51.8M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (2m) 15s ago 21m 61.7M - 2.43.0 a07b618ecd1d c0aa16eafc04 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (19m) 14s ago 19m 75.2M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:13:27.287 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (19m) 15s ago 19m 76.0M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:13:28.045 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:27 smithi114 ceph-mon[23730]: from='client.25309 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:28.045 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:27 smithi114 ceph-mon[28528]: from='client.25309 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:28.069 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ls' 2024-09-13T22:13:28.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:27 smithi121 ceph-mon[24681]: from='client.25309 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:28 smithi114 ceph-mon[23730]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:29.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:28 smithi114 ceph-mon[28528]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:29.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:28 smithi121 ceph-mon[24681]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:29.998 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:29 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:29.998 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:29 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:30.086 INFO:teuthology.orchestra.run.smithi114.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:agent 2/2 18s ago 3m * 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager ?:9093,9094 1/1 17s ago 21m smithi114=a;count:1 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:grafana ?:3000 1/1 18s ago 21m smithi121=a;count:1 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo ?:5000 1/1 17s ago 19m count:1 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:mgr 2/2 18s ago 25m smithi114=y;smithi121=x;count:2 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:mon 3/3 18s ago 25m smithi114:172.21.15.114=a;smithi114:[v2:172.21.15.114:3301,v1:172.21.15.114:6790]=c;smithi121:172.21.15.121=b;count:3 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter ?:9100 2/2 18s ago 21m smithi114=a;smithi121=b;count:2 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:osd 8 18s ago - 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus ?:9095 1/1 18s ago 22m smithi121=a;count:1 2024-09-13T22:13:30.087 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo ?:8000 2/2 18s ago 19m count:2 2024-09-13T22:13:30.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:29 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:30.732 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T22:13:31.035 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:30 smithi114 ceph-mon[28528]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:31.035 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:30 smithi114 ceph-mon[28528]: from='client.15423 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:31.035 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:30 smithi114 ceph-mon[23730]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:31.035 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:30 smithi114 ceph-mon[23730]: from='client.15423 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:30 smithi121 ceph-mon[24681]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:31.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:30 smithi121 ceph-mon[24681]: from='client.15423 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:32.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[28528]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:32.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:32 smithi114 ceph-mon[23730]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:32.848 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:13:32.849 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:13:32.850 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:32.850 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:13:32.850 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:13:32.850 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:13:32.850 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:13:32.850 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:13:32.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:32 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:32 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:32 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:32.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:32 smithi121 ceph-mon[24681]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:33.490 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-13T22:13:33.791 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1071154882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:13:33.791 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.791 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.791 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.791 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1071154882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:13:33.792 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.792 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.792 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:33 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:13:33] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-09-13T22:13:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:33 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1071154882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:13:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:33 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:34.665 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:34 smithi114 ceph-mon[23730]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:34.665 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:34 smithi114 ceph-mon[28528]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:34.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:34 smithi121 ceph-mon[24681]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:37.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:36 smithi114 ceph-mon[23730]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:37.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:36 smithi114 ceph-mon[28528]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:36 smithi121 ceph-mon[24681]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:37.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:37 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:38.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:37 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:38.048 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:37 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:38.825 INFO:teuthology.orchestra.run.smithi114.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:39.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:38 smithi114 ceph-mon[23730]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:39.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:38 smithi114 ceph-mon[23730]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:38 smithi114 ceph-mon[28528]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:39.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:38 smithi114 ceph-mon[28528]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:38 smithi121 ceph-mon[24681]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:38 smithi121 ceph-mon[24681]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:39.517 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T22:13:40.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[28528]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:40.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[28528]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[23730]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:40.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:39 smithi114 ceph-mon[23730]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:39 smithi121 ceph-mon[24681]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:39 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:39 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:13:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:39 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:39 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:39 smithi121 ceph-mon[24681]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:40.943 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:40 smithi114 ceph-mon[23730]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:40.943 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:40 smithi114 ceph-mon[28528]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:41.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:40 smithi121 ceph-mon[24681]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:13:41.860 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:13:42.565 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:13:42.565 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 9s ago 26m - - 2024-09-13T22:13:42.565 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 11s ago 26m - - 2024-09-13T22:13:42.565 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (12m) 9s ago 20m 25.5M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (12m) 10s ago 20m 56.3M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (3m) 9s ago 19m 53.0M - 3.5 e1d6a67b021e 32b02f8593e2 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283 running (15m) 10s ago 25m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 56081b4c7747 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (9m) 9s ago 27m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (27m) 9s ago 27m 78.5M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (25m) 10s ago 25m 61.8M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (25m) 9s ago 25m 56.1M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (12m) 9s ago 21m 15.3M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (12m) 10s ago 21m 14.7M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (25m) 9s ago 25m 56.4M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (24m) 9s ago 24m 56.2M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (24m) 9s ago 24m 49.2M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (24m) 9s ago 24m 50.5M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:13:42.566 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (23m) 10s ago 23m 52.2M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:13:42.567 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (23m) 10s ago 23m 53.9M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:13:42.567 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (22m) 10s ago 22m 51.0M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:13:42.567 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (22m) 10s ago 22m 51.8M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:13:42.567 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (3m) 10s ago 21m 61.7M - 2.43.0 a07b618ecd1d c0aa16eafc04 2024-09-13T22:13:42.567 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (20m) 9s ago 20m 75.2M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:13:42.567 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (20m) 10s ago 20m 76.0M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:13:42.969 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:13:42.969 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:13:42.969 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:13:42.969 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:13:42.970 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:13:42.971 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:13:43.221 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:13:43] ENGINE Bus STOPPING 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: from='client.15441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: Failing over to other MGR 2024-09-13T22:13:43.222 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: from='client.15447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2979183992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: from='client.15441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:13:43.223 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: Failing over to other MGR 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: from='client.15447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T22:13:43.224 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2979183992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:13:43.512 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:43.158+0000 7fb88d4f7640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T22:13:43.512 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ignoring --setuser ceph since I am not root 2024-09-13T22:13:43.512 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ignoring --setgroup ceph since I am not root 2024-09-13T22:13:43.512 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mgr[49218]: -- 172.21.15.121:0/2781978770 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560ffa2c90e0 con 0x560ffa2a7000 2024-09-13T22:13:43.512 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:43.303+0000 7f0552d98040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:13:43.512 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:43.360+0000 7f0552d98040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:13:43.512 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:43.510+0000 7f0552d98040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: from='client.15441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: Failing over to other MGR 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: from='mgr.24947 172.21.15.121:0/1250840702' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:13:43.513 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: from='client.15447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:43.514 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T22:13:43.514 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2979183992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:13:43.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:13:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:13:43.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:13:43] ENGINE Bus STOPPED 2024-09-13T22:13:43.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:13:43] ENGINE Bus STARTING 2024-09-13T22:13:43.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:13:43] ENGINE Serving on http://:::9283 2024-09-13T22:13:43.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:13:43] ENGINE Bus STARTED 2024-09-13T22:13:43.866 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:43.577+0000 7f0552d98040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:13:43.866 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:43.723+0000 7f0552d98040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:13:44.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:43.865+0000 7f0552d98040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:13:44.186 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [ 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr" 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: ], 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "", 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:13:44.187 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:13:44.444 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:44.181+0000 7f0552d98040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:13:44.444 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:44.246+0000 7f0552d98040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:13:44.444 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:44.442+0000 7f0552d98040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.24947 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: mgrmap e35: y(active, starting, since 1.00147s), standbys: x 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:13:44.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: Manager daemon y is now available 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:44.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:13:44.447 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:13:44.447 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:13:44.447 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:13:44.447 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:44.447 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:13:44.468 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.24947 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T22:13:44.468 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: mgrmap e35: y(active, starting, since 1.00147s), standbys: x 2024-09-13T22:13:44.468 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:13:44.468 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:13:44.468 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:13:44.468 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:13:44.468 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:13:44.469 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: Manager daemon y is now available 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:44.470 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.24947 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: mgrmap e35: y(active, starting, since 1.00147s), standbys: x 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:13:44.471 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: Manager daemon y is now available 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:44.472 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:13:44.473 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:13:44.473 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:13:44.473 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:13:44.473 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:44.473 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:13:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:13:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: mgrmap e36: y(active, since 2s), standbys: x 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: from='client.25357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:13:45.271 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: mgrmap e36: y(active, since 2s), standbys: x 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: from='client.25357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-mon[23730]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:13:45.272 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:13:45.149+0000 7f20afa91640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:22:13:45] ENGINE Error in HTTPServer.serve 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._connections.run(self.expiration_interval) 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._run(expiration_interval) 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: s = self.context.wrap_socket( 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:13:45.273 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.sslsocket_class._create( 2024-09-13T22:13:45.274 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:13:45.274 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.do_handshake() 2024-09-13T22:13:45.274 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:13:45.274 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._sslobj.do_handshake() 2024-09-13T22:13:45.274 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:13:45.274 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: mgrmap e36: y(active, since 2s), standbys: x 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: from='client.25357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:13:45.438 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:13:45.439 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:45.439 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-mon[24681]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:13:45.781 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:45.436+0000 7f0552d98040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:13:45.781 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:45.510+0000 7f0552d98040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:13:46.363 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:46.277+0000 7f0552d98040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: [13/Sep/2024:22:13:45] ENGINE Bus STARTING 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: [13/Sep/2024:22:13:45] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: [13/Sep/2024:22:13:45] ENGINE Error in HTTPServer.serve 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:13:46.364 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._run(expiration_interval) 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: s = self.context.wrap_socket( 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.sslsocket_class._create( 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.do_handshake() 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:13:46.365 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._sslobj.do_handshake() 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: [13/Sep/2024:22:13:45] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: [13/Sep/2024:22:13:45] ENGINE Bus STARTED 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:13:46.366 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: mgrmap e37: y(active, since 3s), standbys: x 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.367 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: [13/Sep/2024:22:13:45] ENGINE Bus STARTING 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: [13/Sep/2024:22:13:45] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: [13/Sep/2024:22:13:45] ENGINE Error in HTTPServer.serve 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:13:46.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.do_handshake() 2024-09-13T22:13:46.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: [13/Sep/2024:22:13:45] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: [13/Sep/2024:22:13:45] ENGINE Bus STARTED 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:13:46.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: [13/Sep/2024:22:13:45] ENGINE Bus STARTING 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: [13/Sep/2024:22:13:45] ENGINE Serving on https://172.21.15.114:7150 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: [13/Sep/2024:22:13:45] ENGINE Error in HTTPServer.serve 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:13:46.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.do_handshake() 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:13:46.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: [13/Sep/2024:22:13:45] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: [13/Sep/2024:22:13:45] ENGINE Bus STARTED 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:46.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: mgrmap e37: y(active, since 3s), standbys: x 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.582 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: mgrmap e37: y(active, since 3s), standbys: x 2024-09-13T22:13:46.582 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.582 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.582 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:46.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:46.399+0000 7f0552d98040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:13:46.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:46.545+0000 7f0552d98040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:13:46.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:46.610+0000 7f0552d98040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:13:46.912 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:46.768+0000 7f0552d98040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:13:46.912 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:46.840+0000 7f0552d98040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:13:47.176 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:46.910+0000 7f0552d98040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:13:47.176 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:47.106+0000 7f0552d98040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:13:47.462 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:47.177+0000 7f0552d98040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:13:47.463 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:47.255+0000 7f0552d98040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:13:47.463 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:47.338+0000 7f0552d98040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:13:47.463 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:47.398+0000 7f0552d98040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:13:47.463 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:47.461+0000 7f0552d98040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:13:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.224 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:48.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.225 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:48.282+0000 7f0552d98040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:13:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:13:48] ENGINE Bus STARTING 2024-09-13T22:13:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: CherryPy Checker: 2024-09-13T22:13:48.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: The Application mounted at '' has an empty config. 2024-09-13T22:13:48.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: 2024-09-13T22:13:48.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:13:48] ENGINE Serving on http://:::9283 2024-09-13T22:13:48.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: [13/Sep/2024:22:13:48] ENGINE Bus STARTED 2024-09-13T22:13:49.198 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: mgrmap e38: y(active, since 5s), standbys: x 2024-09-13T22:13:49.198 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.198 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: Standby manager daemon x restarted 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: mgrmap e38: y(active, since 5s), standbys: x 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.199 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.200 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: Standby manager daemon x restarted 2024-09-13T22:13:49.200 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T22:13:49.200 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:13:49.200 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:13:49.200 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:13:49.200 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:48 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:13:49.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: mgrmap e38: y(active, since 5s), standbys: x 2024-09-13T22:13:49.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:49.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: Standby manager daemon x restarted 2024-09-13T22:13:49.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T22:13:49.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:13:49.239 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:13:49.239 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:13:49.239 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:48 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2865562662' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:13:50.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:50 smithi121 ceph-mon[24681]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:50.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:50 smithi121 ceph-mon[24681]: mgrmap e39: y(active, since 6s), standbys: x 2024-09-13T22:13:50.364 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:50 smithi121 ceph-mon[24681]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:50.420 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:50 smithi114 ceph-mon[23730]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:50.421 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:50 smithi114 ceph-mon[23730]: mgrmap e39: y(active, since 6s), standbys: x 2024-09-13T22:13:50.421 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:50 smithi114 ceph-mon[23730]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:50.421 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:50 smithi114 ceph-mon[28528]: from='client.25261 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:13:50.421 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:50 smithi114 ceph-mon[28528]: mgrmap e39: y(active, since 6s), standbys: x 2024-09-13T22:13:50.421 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:50 smithi114 ceph-mon[28528]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:13:51.474+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:51.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:51.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:51.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:13:51.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:13:51.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:13:51.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:13:51.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:13:51.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:13:51.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:51.820 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:13:51.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:51.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:51.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:51.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:51.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:51.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:13:51.821 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:51.822 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:13:51.822 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:51.822 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:13:51.822 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:51.822 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:51.822 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:51.823 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:51.823 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:51.823 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:51.823 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:51.823 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:51.823 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:52.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:52 smithi121 ceph-mon[24681]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:13:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:52 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:13:53.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:53.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:53.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:53.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:53.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:53.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:52 smithi121 ceph-mon[24681]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:13:53.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:52 smithi121 ceph-mon[24681]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:13:53.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[28528]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:13:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[23730]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:13:53.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:53.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:13:53.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:13:53.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:13:53.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:13:53.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:13:53.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:13:53.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:13:53.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:13:53.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:13:53.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:13:53.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:13:53.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:13:53.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:13:53.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:53.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:13:53.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.334 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[23730]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:13:53.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:13:53.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[28528]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[28528]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:13:53.337 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:52 smithi114 ceph-mon[23730]: Reconfiguring daemon prometheus.a on smithi121 2024-09-13T22:13:53.818 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:53 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[49214]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T22:13:54.573 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-mon[24681]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:13:54.573 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.573 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.573 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[23730]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:13:54.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[28528]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T22:13:54.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:54 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:54.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 systemd[1]: Stopping Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:13:55.227 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T22:13:55.227 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[66834]: ts=2024-09-13T22:13:54.968Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T22:13:55.228 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:54 smithi121 podman[71122]: 2024-09-13 22:13:54.978346662 +0000 UTC m=+0.110466547 container died c0aa16eafc042269120313b482e0f3d5e52c613f394a4f93272ce2970dea5d6a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:13:55.603 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 podman[71122]: 2024-09-13 22:13:55.342701807 +0000 UTC m=+0.474821692 container cleanup c0aa16eafc042269120313b482e0f3d5e52c613f394a4f93272ce2970dea5d6a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:13:55.603 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 bash[71122]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T22:13:55.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 podman[71137]: 2024-09-13 22:13:55.602377887 +0000 UTC m=+0.624965062 container remove c0aa16eafc042269120313b482e0f3d5e52c613f394a4f93272ce2970dea5d6a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:13:55.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T22:13:55.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 systemd[1]: Stopped Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:13:55.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.673s CPU time. 2024-09-13T22:13:55.872 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 systemd[1]: Starting Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:13:56.208 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 podman[71226]: 2024-09-13 22:13:55.859644182 +0000 UTC m=+0.016943320 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T22:13:56.208 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:55 smithi121 podman[71226]: 2024-09-13 22:13:55.970287253 +0000 UTC m=+0.127586446 container create 7738fb682bbc3a159119d019f643a6f977b35f69915455eaed7f14dd3c2fa79c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:13:56.208 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 podman[71226]: 2024-09-13 22:13:56.181506187 +0000 UTC m=+0.338805318 container init 7738fb682bbc3a159119d019f643a6f977b35f69915455eaed7f14dd3c2fa79c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:13:56.208 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 podman[71226]: 2024-09-13 22:13:56.184180277 +0000 UTC m=+0.341479409 container start 7738fb682bbc3a159119d019f643a6f977b35f69915455eaed7f14dd3c2fa79c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:13:56.621 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.207Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.207Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.207Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi121 (none))" 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.207Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.207Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.209Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.209Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.211Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.211Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.213Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.214Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.500994ms 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.214Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.214Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-09-13T22:13:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.232Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.243Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.245Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.247Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.247Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.247Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.793µs wal_replay_duration=33.378995ms wbl_replay_duration=97ns total_replay_duration=34.934851ms 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.251Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.251Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.251Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 bash[71226]: 7738fb682bbc3a159119d019f643a6f977b35f69915455eaed7f14dd3c2fa79c 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 systemd[1]: Started Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.272Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.78131ms db_storage=1.329µs remote_storage=1.53µs web_handler=430ns query_engine=821ns scrape=463.716µs scrape_sd=171.278µs notify=22.333µs notify_sd=20.965µs rules=19.325574ms tracing=6.39µs 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.272Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T22:13:56.623 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:13:56.272Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T22:13:57.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-mon[24681]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:13:57.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:57.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:57.031 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:13:57.032 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:13:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[23730]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:13:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:57.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:13:57.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:13:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[28528]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-13T22:13:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:13:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:13:57.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:13:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:13:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:13:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:13:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:58.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:13:58.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:13:58.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:13:58.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:57 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:58.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:57 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:13:58.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:57 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:13:58.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:57 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:13:58.098 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:57 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:13:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[23730]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:13:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[23730]: Upgrade: Updating mgr.x 2024-09-13T22:13:59.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[23730]: Deploying daemon mgr.x on smithi121 2024-09-13T22:13:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[28528]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:13:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[28528]: Upgrade: Updating mgr.x 2024-09-13T22:13:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[28528]: Deploying daemon mgr.x on smithi121 2024-09-13T22:13:59.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:13:58 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:58 smithi121 ceph-mon[24681]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:13:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:58 smithi121 ceph-mon[24681]: Upgrade: Updating mgr.x 2024-09-13T22:13:59.087 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:58 smithi121 ceph-mon[24681]: Deploying daemon mgr.x on smithi121 2024-09-13T22:13:59.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:13:58 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:13:59.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:59 smithi121 systemd[1]: Stopping Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:13:59.917 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:13:59 smithi121 podman[71567]: 2024-09-13 22:13:59.639038961 +0000 UTC m=+0.132767013 container died 56081b4c77470436765fc94d3dfe84d5aba0c513957d76556ae8d37988eb2b19 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2) 2024-09-13T22:14:00.243 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 podman[71567]: 2024-09-13 22:14:00.016438672 +0000 UTC m=+0.510166731 container cleanup 56081b4c77470436765fc94d3dfe84d5aba0c513957d76556ae8d37988eb2b19 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0) 2024-09-13T22:14:00.243 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 bash[71567]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x 2024-09-13T22:14:00.581 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 podman[71579]: 2024-09-13 22:14:00.2428117 +0000 UTC m=+0.600289795 container remove 56081b4c77470436765fc94d3dfe84d5aba0c513957d76556ae8d37988eb2b19 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:14:00.581 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:14:00.581 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T22:14:00.581 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 systemd[1]: Stopped Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:14:00.581 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Consumed 39.972s CPU time. 2024-09-13T22:14:00.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:00 smithi121 ceph-mon[24681]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:14:00.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 systemd[1]: Starting Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:14:01.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:00 smithi114 ceph-mon[23730]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:14:01.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:00 smithi114 ceph-mon[28528]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:14:01.155 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 podman[71705]: 2024-09-13 22:14:00.786621516 +0000 UTC m=+0.016058220 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:14:01.155 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:00 smithi121 podman[71705]: 2024-09-13 22:14:00.902021642 +0000 UTC m=+0.131458378 container create 0db02b1df602214dfde0746f04656f2c83770d4692c4b903d9160f0eb5817c69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T22:14:01.504 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 podman[71705]: 2024-09-13 22:14:01.155099099 +0000 UTC m=+0.384535803 container init 0db02b1df602214dfde0746f04656f2c83770d4692c4b903d9160f0eb5817c69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_CLEAN=True) 2024-09-13T22:14:01.504 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 podman[71705]: 2024-09-13 22:14:01.158020387 +0000 UTC m=+0.387457079 container start 0db02b1df602214dfde0746f04656f2c83770d4692c4b903d9160f0eb5817c69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:14:01.504 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 ceph-mgr[71724]: -- 172.21.15.121:0/2771572249 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ebf15bd0e0 con 0x55ebf159a800 2024-09-13T22:14:01.504 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 bash[71705]: 0db02b1df602214dfde0746f04656f2c83770d4692c4b903d9160f0eb5817c69 2024-09-13T22:14:01.505 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 systemd[1]: Started Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:14:01.505 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:01.277+0000 7f745fa45040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:14:01.505 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:01.351+0000 7f745fa45040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:14:01.505 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:01.502+0000 7f745fa45040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:14:01.842 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:14:01] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-09-13T22:14:01.856 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:01.568+0000 7f745fa45040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:14:01.857 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:01.713+0000 7f745fa45040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:14:01.857 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:01 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:01.855+0000 7f745fa45040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:14:02.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:14:02.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:02.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:02.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:02.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:02.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:02.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:02.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:02.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:14:02.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:02.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:02.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:02.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:02.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:02.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:02.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:02 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:02.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:02.167+0000 7f745fa45040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:14:02.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:02.226+0000 7f745fa45040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:14:02.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:02.411+0000 7f745fa45040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:02.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:02 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:03.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:03.302+0000 7f745fa45040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:14:03.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:03 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:03.369+0000 7f745fa45040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:14:04.336 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.085+0000 7f745fa45040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:14:04.336 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.199+0000 7f745fa45040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:14:04.377 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-mon[23730]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:04.377 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2759237695' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:14:04.378 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-mon[28528]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:04.378 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2759237695' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:14:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-mon[24681]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2759237695' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-13T22:14:04.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.341+0000 7f745fa45040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:14:04.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.402+0000 7f745fa45040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:14:04.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.558+0000 7f745fa45040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:14:04.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:04.411+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:04.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:04.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:04.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:04.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:04.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:04.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:04.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:04.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:04.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:04.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:04.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:04.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:04.911 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.627+0000 7f745fa45040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:14:04.911 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.687+0000 7f745fa45040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:14:04.911 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.845+0000 7f745fa45040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:14:05.178 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.910+0000 7f745fa45040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:14:05.178 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:04 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:04.976+0000 7f745fa45040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:14:05.178 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:05.058+0000 7f745fa45040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:14:05.178 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:05.118+0000 7f745fa45040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:14:05.178 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:05.177+0000 7f745fa45040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:05.516 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:05.517 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:05.518 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.519 INFO:journalctl@ceph.mon.b.smithi121.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:05.520 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all mgr 2024-09-13T22:14:05.521 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:05.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:05 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:05.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:05.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:05.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:05.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:05.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:05.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:05.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:05.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:05.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:05.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:05.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:05.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:05.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:05.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:05.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:05.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:05.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:05.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:05.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:05.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:05.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:05.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:05.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:05.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:05.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:05.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:05.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:05.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:05.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:05.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:05.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:05.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:05.584 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:05.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:05.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi-foo-smithi114-lemsof 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: /usr/bin/podman: stderr Error: no such container ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:05.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all mgr 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:05.590 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:05.591 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all mgr 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:05.592 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:05 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:06.263 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:06.004+0000 7f745fa45040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:14:06.263 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: [13/Sep/2024:22:14:06] ENGINE Bus STARTING 2024-09-13T22:14:06.263 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: CherryPy Checker: 2024-09-13T22:14:06.263 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: The Application mounted at '' has an empty config. 2024-09-13T22:14:06.263 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: 2024-09-13T22:14:06.264 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: [13/Sep/2024:22:14:06] ENGINE Serving on http://:::9283 2024-09-13T22:14:06.264 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[71720]: [13/Sep/2024:22:14:06] ENGINE Bus STARTED 2024-09-13T22:14:06.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:06.020+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:06.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:06.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:06.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:06.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:06.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:06.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:06.331 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:06.331 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:06.331 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:06.331 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:06.864 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:14:06.864 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: Standby manager daemon x restarted 2024-09-13T22:14:06.864 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T22:14:06.864 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:06.864 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:06.865 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:06 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:14:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: Standby manager daemon x restarted 2024-09-13T22:14:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T22:14:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:07.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: Standby manager daemon x restarted 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/1366668563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:07.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:07.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:07.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:07.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:07.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:07.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:07.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:06 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:07.808 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:07.637+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:07.808 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:07.808 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:07.808 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:07.808 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:07.808 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:07.808 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:07.809 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:07.810 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:07.811 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:07.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:08.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:08.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:08.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:08.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: mgrmap e40: y(active, since 24s), standbys: x 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:08.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: mgrmap e40: y(active, since 24s), standbys: x 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:08.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:07 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: mgrmap e40: y(active, since 24s), standbys: x 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:08.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:07 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:08.914 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:08 smithi121 ceph-mon[24681]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:08.914 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:08 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.914 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.914 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.914 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.914 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.915 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:08.916 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:08.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:08 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:08 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:08.918 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:08 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[23730]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:09.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[28528]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:09.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:09.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:09.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:09.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:09.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:09.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:09.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:09.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:09.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:09.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:08 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:10.963 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:10.964 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:10.964 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:10.964 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:10.964 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:10.964 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:10.964 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:10.964 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:11.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:11.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:11.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:11.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:11.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:10.628+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:11.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:11.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:11.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:11.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:11.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:11.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:11.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:11.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:14:11] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-09-13T22:14:11.898 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:11.898 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:11.898 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:11.899 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:11.900 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.901 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:11.902 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:11.903 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.904 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:11.905 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:11.906 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:11.907 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:11.908 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:11.909 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:11.909 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:11.909 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:11.909 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:11 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:12.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:12.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:12.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:12.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:11 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:12.128 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:11 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:12.128 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:11 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:12.128 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:11 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:12.128 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:11 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:12.128 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:11 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:12.895 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-mon[23730]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T22:14:12.895 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:12.895 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-mon[28528]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T22:14:12.895 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:12.896 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:12.689+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:12.896 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:12.896 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:12.896 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:12.896 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:12.896 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.896 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:12.897 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:12.898 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:12.899 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:12.900 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.082 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:12 smithi121 ceph-mon[24681]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T22:14:13.082 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:13.962 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:13.963 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:13.964 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:13.965 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:13.966 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:13.967 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:13.968 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:13.969 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:13.970 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:13.971 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.972 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.972 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:13 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.972 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:13 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:13.972 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:13.972 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:13.973 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:13.974 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:13.975 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:13.976 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:13.977 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:13.977 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:13 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:13.977 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:13 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.977 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:13 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:13.977 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:13 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:14.556 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:14:14.981 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:14 smithi121 ceph-mon[24681]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T22:14:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:14 smithi114 ceph-mon[23730]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T22:14:15.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:14 smithi114 ceph-mon[28528]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-13T22:14:15.229 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 1s ago 26m - - 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 4s ago 26m - - 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (13m) 1s ago 21m 25.5M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (12m) 4s ago 21m 57.6M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 running (3m) 1s ago 20m 53.2M - 3.5 e1d6a67b021e 32b02f8593e2 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283 running (16m) 4s ago 26m 334M - 19.1.1-324-gecab7be5 0652ff2b5d78 56081b4c7747 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (9m) 1s ago 27m 497M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (27m) 1s ago 27m 81.0M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (26m) 4s ago 26m 65.5M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (26m) 1s ago 26m 60.7M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (13m) 1s ago 22m 14.7M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:14:15.230 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (12m) 4s ago 22m 14.8M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (25m) 1s ago 25m 57.4M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (25m) 1s ago 25m 57.5M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (25m) 1s ago 25m 50.5M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (24m) 1s ago 24m 51.9M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (24m) 4s ago 24m 54.5M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (23m) 4s ago 23m 54.6M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (23m) 4s ago 23m 52.3M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (23m) 4s ago 23m 53.3M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (3m) 4s ago 22m 61.7M - 2.43.0 a07b618ecd1d c0aa16eafc04 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (20m) 1s ago 20m 75.7M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:14:15.231 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (20m) 4s ago 20m 76.3M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:14:15.643 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:14:15.643 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:14:15.643 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:14:15.643 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:15.643 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:14:15.644 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:14:15.986 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:14:15.986 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [ 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr" 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: ], 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:14:15.987 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:14:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:15 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:15 smithi121 ceph-mon[24681]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:15 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:15 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:15 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:15 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3214123286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:16.205 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:16.205 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[28528]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:16.205 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.205 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.205 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3214123286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:16.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:16.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[23730]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:16.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:15 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3214123286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:16.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:16.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:17.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:17.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:17.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:14:17.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:14:17.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:17.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:17.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:17.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:17.918 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:17 smithi121 ceph-mon[24681]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:17.918 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:17 smithi121 ceph-mon[24681]: Upgrade: Updating mgr.x 2024-09-13T22:14:17.918 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:17 smithi121 ceph-mon[24681]: Deploying daemon mgr.x on smithi121 2024-09-13T22:14:17.919 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:17 smithi121 ceph-mon[24681]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 898 B/s rd, 0 op/s 2024-09-13T22:14:17.919 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:17 smithi121 ceph-mon[24681]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:14:18.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[23730]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:18.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[23730]: Upgrade: Updating mgr.x 2024-09-13T22:14:18.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[23730]: Deploying daemon mgr.x on smithi121 2024-09-13T22:14:18.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[23730]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 898 B/s rd, 0 op/s 2024-09-13T22:14:18.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[23730]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:14:18.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[28528]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:18.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[28528]: Upgrade: Updating mgr.x 2024-09-13T22:14:18.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[28528]: Deploying daemon mgr.x on smithi121 2024-09-13T22:14:18.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[28528]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 898 B/s rd, 0 op/s 2024-09-13T22:14:18.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:17 smithi114 ceph-mon[28528]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:14:18.566 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:18 smithi121 systemd[1]: Stopping Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:14:18.859 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:18 smithi121 podman[73313]: 2024-09-13 22:14:18.564468247 +0000 UTC m=+0.121025810 container died 0db02b1df602214dfde0746f04656f2c83770d4692c4b903d9160f0eb5817c69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T22:14:19.118 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:18 smithi121 podman[73313]: 2024-09-13 22:14:18.858635386 +0000 UTC m=+0.415192993 container cleanup 0db02b1df602214dfde0746f04656f2c83770d4692c4b903d9160f0eb5817c69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T22:14:19.119 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:18 smithi121 bash[73313]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x 2024-09-13T22:14:19.119 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.119 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.119 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:18 smithi121 ceph-mon[24681]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s 2024-09-13T22:14:19.119 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:18 smithi121 ceph-mon[24681]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:19.119 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[23730]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[23730]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[28528]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[28528]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:19.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 podman[73325]: 2024-09-13 22:14:19.118245707 +0000 UTC m=+0.553747469 container remove 0db02b1df602214dfde0746f04656f2c83770d4692c4b903d9160f0eb5817c69 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T22:14:19.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:14:19.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T22:14:19.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 systemd[1]: Stopped Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:14:19.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Consumed 5.775s CPU time. 2024-09-13T22:14:19.686 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 systemd[1]: Starting Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:14:19.956 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 podman[73452]: 2024-09-13 22:14:19.589487856 +0000 UTC m=+0.016746602 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:14:19.956 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 podman[73452]: 2024-09-13 22:14:19.694763049 +0000 UTC m=+0.122021788 container create 45663949b7f3960b88995740a68765b46fbbc6e03f217c79103734ac75789795 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, RELEASE=HEAD, version=8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 podman[73452]: 2024-09-13 22:14:19.955893643 +0000 UTC m=+0.383152382 container init 45663949b7f3960b88995740a68765b46fbbc6e03f217c79103734ac75789795 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, com.redhat.component=centos-stream-container, name=centos-stream, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 podman[73452]: 2024-09-13 22:14:19.959051868 +0000 UTC m=+0.386310604 container start 45663949b7f3960b88995740a68765b46fbbc6e03f217c79103734ac75789795 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, release=754, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:19 smithi121 ceph-mgr[73470]: -- 172.21.15.121:0/3671481948 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559373650340 con 0x559373629000 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 bash[73452]: 45663949b7f3960b88995740a68765b46fbbc6e03f217c79103734ac75789795 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 systemd[1]: Started Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:20.105+0000 7fb0ec001000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:20.188+0000 7fb0ec001000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:14:20.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:20.366+0000 7fb0ec001000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:14:20.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:20.454+0000 7fb0ec001000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:14:20.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:20.584+0000 7fb0ec001000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:14:21.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:20.795+0000 7fb0ec001000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:21.281 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:21.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:21.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:21.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:21.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:21.390 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:21.127+0000 7fb0ec001000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:14:21.390 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:21.216+0000 7fb0ec001000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:21.391 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:21.699 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:21.389+0000 7fb0ec001000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:14:21.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:21.680+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:21.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:21.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:21.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:21.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:21.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:14:21] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.43.0" 2024-09-13T22:14:22.459 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:22.460 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:22.461 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:22.462 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:22.463 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:22.464 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:22.465 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:22.465 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:22.465 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:22.465 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:22.465 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.510 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.511 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:22.512 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:22.513 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:22.514 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:22.515 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:22.516 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:22.517 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:22.518 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:22.519 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-13T22:14:22.520 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:22.521 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:22.522 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:22.522 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:22.522 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:22.522 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:22.522 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:22 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:22.734 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:22.493+0000 7fb0ec001000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:14:22.735 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:22.589+0000 7fb0ec001000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:14:23.540 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:23.541 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:23.223+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:23.542 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:23.543 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:23.544 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:23.545 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:23.546 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:23.547 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:23.547 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:23.624 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:23.547+0000 7fb0ec001000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:14:23.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:23 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:23.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:23 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:23.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:23 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:23.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:23 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:23.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:23 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:23.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:14:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:14:23.540Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:14:23.976 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:23.877+0000 7fb0ec001000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:14:24.241 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:24.033+0000 7fb0ec001000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:14:24.241 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:24.131+0000 7fb0ec001000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:14:24.496 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:24.238+0000 7fb0ec001000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:14:24.496 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:24.338+0000 7fb0ec001000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:14:24.497 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.497 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:24.497 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.497 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.498 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:24.499 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:24.500 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.501 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.502 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:24.502 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.502 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.502 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.502 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.502 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.503 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.503 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.503 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.503 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.503 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-09-13T22:14:24.503 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:24.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:24.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:24.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:24.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:24.504 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:24.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:24.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:24.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:24.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:24.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-09-13T22:14:24.584 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:24.585 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:24.586 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:24.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:24.495+0000 7fb0ec001000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:14:24.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:24.597+0000 7fb0ec001000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:14:24.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:24.705+0000 7fb0ec001000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:24.793+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.964 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:24.965 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:24.966 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:24.967 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.284 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:25.002+0000 7fb0ec001000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:14:25.284 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:25.092+0000 7fb0ec001000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:14:25.285 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:25.183+0000 7fb0ec001000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.560 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.561 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:25.562 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:25.563 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:25.564 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:25.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:25.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:25.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:25.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:25.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:25.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:25 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:25.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:25.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:25.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:25.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:25.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:25.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:25.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:25.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:25.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:25.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:25.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:25.588 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:25.589 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:25.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:25.591 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:25.592 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:25.593 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:25.594 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:25.595 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:25.595 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:25.595 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:25.595 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:25.595 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:25 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:26.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:14:26.539Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-09-13T22:14:26.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:26.417+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:26.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:26.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:26.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:26.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:26.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:26.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:26.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:26.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:26.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:26.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:26.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:26.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:26.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:26.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:26.285+0000 7fb0ec001000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:14:26.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: [13/Sep/2024:22:14:26] ENGINE Bus STARTING 2024-09-13T22:14:26.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: CherryPy Checker: 2024-09-13T22:14:26.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: The Application mounted at '' has an empty config. 2024-09-13T22:14:26.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: 2024-09-13T22:14:26.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: [13/Sep/2024:22:14:26] ENGINE Serving on http://:::9283 2024-09-13T22:14:26.622 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[73466]: [13/Sep/2024:22:14:26] ENGINE Bus STARTED 2024-09-13T22:14:27.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:27.003 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: Standby manager daemon x restarted 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:27.004 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:27.073 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:27.073 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:27.073 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: Standby manager daemon x restarted 2024-09-13T22:14:27.073 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T22:14:27.073 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:27.073 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:27.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:27.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:27.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:27.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:27.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:27.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:27.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: Standby manager daemon x restarted 2024-09-13T22:14:27.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:27.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:28.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:28.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:28.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:28.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:28.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:28.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:28.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:28.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:28.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:28.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:28.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:28.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:28.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:28.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:28.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:28.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:28.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[28528]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:28.083 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[28528]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[23730]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[23730]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[28528]: mgrmap e41: y(active, since 44s), standbys: x 2024-09-13T22:14:28.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:28.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:28.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:28.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-mon[23730]: mgrmap e41: y(active, since 44s), standbys: x 2024-09-13T22:14:28.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:27.873+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:28.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.092 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:27 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:28.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:28.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:27 smithi121 ceph-mon[24681]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:27 smithi121 ceph-mon[24681]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:27 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:27 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:27 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:28.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:27 smithi121 ceph-mon[24681]: mgrmap e41: y(active, since 44s), standbys: x 2024-09-13T22:14:29.059 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:29.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:29.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:29.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:29.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:29.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:29.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:29.515+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.812 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:29.813 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:29.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:29.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:29.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:29.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:29.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:29.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:29.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:29.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:29.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:29.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:29.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:29.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:29.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:29.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:29.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:29.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:29.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:29.877 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:29.877 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:29.877 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:29.877 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:29.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:29.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:29.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:29.878 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:29.878 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:29.878 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:29.878 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:29.878 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:29.878 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:29.879 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:29.879 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:29.879 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:29.879 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:29 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:30.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:30.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:30.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:30.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:30.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:30.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:30.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:30.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:30.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:30.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:30.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:30.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:30.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:30.866 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:30 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:30.867 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:30.868 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:30.869 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:30.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:30 smithi121 ceph-mon[24681]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:30 smithi121 ceph-mon[24681]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:30 smithi121 ceph-mon[24681]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:30 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:30 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:30 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:31.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:31.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:31.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:31.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:31.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:31.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:31.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[23730]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[23730]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[23730]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:31.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:31.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[28528]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[28528]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:31.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[28528]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:31.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:31.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:31.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:30 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:14:31] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:31.839+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.060 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:33.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:33.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:33.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:33.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:33.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:32 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:33.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:33.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:33.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:33.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:33.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:33.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:33.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:33.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:33.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:32 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:34.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[28528]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:34.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[28528]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[28528]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[23730]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[23730]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[23730]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:34.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:33 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:34.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:33 smithi121 ceph-mon[24681]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:34.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:33 smithi121 ceph-mon[24681]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-09-13T22:14:34.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:33 smithi121 ceph-mon[24681]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:34.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:33 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:34.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:33 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:34.088 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:33 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:34.960 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 858 B/s rd, 0 op/s 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 op/s 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 858 B/s rd, 0 op/s 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.962 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 op/s 2024-09-13T22:14:34.963 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.201 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 858 B/s rd, 0 op/s 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:35.202 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.203 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.203 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.203 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-09-13T22:14:35.203 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 op/s 2024-09-13T22:14:35.203 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:35.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:35.001+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:35.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:35.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:35.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:35.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:35.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:35.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:35.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:35.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:35.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:35.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:35.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:35.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:35.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:35.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:35.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:35.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:35.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:35.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:35.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:35.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:35 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:36.250 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.251 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:36.252 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:36.253 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.254 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:35 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:36.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:36.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:36.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:36.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:36.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:36.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:36.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:36.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:36.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:36.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:36.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.331 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:36.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:36.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:36.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:36.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:36.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:36.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:36.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:36.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:36.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:36.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:36.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:36.336 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:37.207 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:36 smithi121 ceph-mon[24681]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-09-13T22:14:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:36 smithi114 ceph-mon[23730]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-09-13T22:14:37.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:36 smithi114 ceph-mon[28528]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-09-13T22:14:39.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[23730]: Upgrade: Updating mgr.x 2024-09-13T22:14:39.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[23730]: Deploying daemon mgr.x on smithi121 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[23730]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[28528]: Upgrade: Updating mgr.x 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:14:39.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:39.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[28528]: Deploying daemon mgr.x on smithi121 2024-09-13T22:14:39.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:39 smithi114 ceph-mon[28528]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:39 smithi121 ceph-mon[24681]: Upgrade: Updating mgr.x 2024-09-13T22:14:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:39.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:14:39.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:14:39.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:39.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:39 smithi121 ceph-mon[24681]: Deploying daemon mgr.x on smithi121 2024-09-13T22:14:39.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:39 smithi121 ceph-mon[24681]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:40.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:39 smithi121 systemd[1]: Stopping Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:14:40.453 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 podman[75363]: 2024-09-13 22:14:40.191274829 +0000 UTC m=+0.128526521 container died 45663949b7f3960b88995740a68765b46fbbc6e03f217c79103734ac75789795 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, distribution-scope=public, GIT_BRANCH=HEAD, RELEASE=HEAD, io.buildah.version=1.19.8, version=8, ceph=True, name=centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=) 2024-09-13T22:14:40.803 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 podman[75363]: 2024-09-13 22:14:40.534939287 +0000 UTC m=+0.472190980 container cleanup 45663949b7f3960b88995740a68765b46fbbc6e03f217c79103734ac75789795 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, version=8, architecture=x86_64, io.buildah.version=1.19.8, vendor=Red Hat, Inc., distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, maintainer=Guillaume Abrioux , name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T22:14:40.803 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 bash[75363]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x 2024-09-13T22:14:41.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 podman[75375]: 2024-09-13 22:14:40.802987299 +0000 UTC m=+0.609315905 container remove 45663949b7f3960b88995740a68765b46fbbc6e03f217c79103734ac75789795 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux ) 2024-09-13T22:14:41.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:14:41.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T22:14:41.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 systemd[1]: Stopped Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:14:41.122 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:40 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service: Consumed 7.736s CPU time. 2024-09-13T22:14:41.492 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 systemd[1]: Starting Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:14:41.492 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 podman[75503]: 2024-09-13 22:14:41.255461544 +0000 UTC m=+0.016018828 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:14:41.492 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 podman[75503]: 2024-09-13 22:14:41.378944308 +0000 UTC m=+0.139501580 container create 9dc33ce81ccf0c4e40d8c9b960a342d7719e6ae8a4ebe34b65794cb93eb38f1b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T22:14:41.800 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 podman[75503]: 2024-09-13 22:14:41.615044601 +0000 UTC m=+0.375601872 container init 9dc33ce81ccf0c4e40d8c9b960a342d7719e6ae8a4ebe34b65794cb93eb38f1b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:14:41.800 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 podman[75503]: 2024-09-13 22:14:41.617863297 +0000 UTC m=+0.378420569 container start 9dc33ce81ccf0c4e40d8c9b960a342d7719e6ae8a4ebe34b65794cb93eb38f1b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T22:14:41.800 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-mgr[75522]: -- 172.21.15.121:0/4186617127 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5580961790e0 con 0x558096157000 2024-09-13T22:14:41.800 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 bash[75503]: 9dc33ce81ccf0c4e40d8c9b960a342d7719e6ae8a4ebe34b65794cb93eb38f1b 2024-09-13T22:14:41.800 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 systemd[1]: Started Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:14:41.800 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:41.737+0000 7f1bc743c040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:14:41.800 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:41.797+0000 7f1bc743c040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:14:42.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:14:41] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-09-13T22:14:42.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[28528]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:14:42.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[23730]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:41.951+0000 7f1bc743c040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:14:42.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:42.017+0000 7f1bc743c040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:14:42.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-mon[24681]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:14:42.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:41 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:42.617 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:42.165+0000 7f1bc743c040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:14:42.617 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:42.303+0000 7f1bc743c040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:14:42.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[23730]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:42.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[23730]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 689 B/s rd, 0 op/s 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[28528]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[28528]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 689 B/s rd, 0 op/s 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:42.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:42.869 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:42.614+0000 7f1bc743c040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:14:42.869 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:42.677+0000 7f1bc743c040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:14:42.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-mon[24681]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:14:42.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-mon[24681]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 689 B/s rd, 0 op/s 2024-09-13T22:14:42.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:42.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:42.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:43.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:42 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:42.867+0000 7f1bc743c040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:14:43.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:43.332+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:43.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:43.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:43.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:43.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:43.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:43.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:43.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:43.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:43.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:43.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:43.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:43.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:43.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:43.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:43.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:43.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:43.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:43.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:43.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:43.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:44.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:44.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:44.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:44.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:44.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:44.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:43.757+0000 7f1bc743c040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:14:44.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:43.840+0000 7f1bc743c040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:14:44.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:44.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:44.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:44.830 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:44.573+0000 7f1bc743c040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:14:44.830 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:44.694+0000 7f1bc743c040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:14:44.830 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:44.831 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:44.832 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:44.833 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:44.834 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-mon[24681]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-mon[24681]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-mon[24681]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:44.835 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:45.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[28528]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[28528]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[23730]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[23730]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[28528]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-09-13T22:14:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[23730]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-09-13T22:14:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:44.847+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:45.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:45.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:45.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:45.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:44.851+0000 7f1bc743c040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:44 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:44.913+0000 7f1bc743c040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:14:45.090 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.067+0000 7f1bc743c040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:14:45.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.141+0000 7f1bc743c040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:14:45.368 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.203+0000 7f1bc743c040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:14:45.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.365+0000 7f1bc743c040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:14:45.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.434+0000 7f1bc743c040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:14:45.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.502+0000 7f1bc743c040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:14:45.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.589+0000 7f1bc743c040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:14:46.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:46.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:46.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:46.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:46.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:46.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:46.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:46.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:46.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:46.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:46.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.657+0000 7f1bc743c040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:14:46.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:45.732+0000 7f1bc743c040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:46.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:46.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:46.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:46.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:46.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:46.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:46.345 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:46.510+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:46.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:46.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:46.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:46.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:46.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:46.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:46.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:46.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:46.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:46.612+0000 7f1bc743c040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:14:46.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: [13/Sep/2024:22:14:46] ENGINE Bus STARTING 2024-09-13T22:14:46.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: CherryPy Checker: 2024-09-13T22:14:46.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: The Application mounted at '' has an empty config. 2024-09-13T22:14:46.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: [13/Sep/2024:22:14:46] ENGINE Serving on http://:::9283 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: [13/Sep/2024:22:14:46] ENGINE Bus STARTED 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: Standby manager daemon x restarted 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: Standby manager daemon x started 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:47.052 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:14:47.052 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 12s ago 27m - - 2024-09-13T22:14:47.052 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 12s ago 27m - - 2024-09-13T22:14:47.052 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (13m) 12s ago 21m 25.5M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:14:47.052 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (13m) 12s ago 21m 56.5M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:14:47.052 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 12s ago 20m 46.2M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 starting - - - - 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (10m) 12s ago 28m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (28m) 12s ago 28m 82.7M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (26m) 12s ago 26m 69.5M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (27m) 12s ago 27m 61.5M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (14m) 12s ago 22m 14.4M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (13m) 12s ago 22m 14.8M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (26m) 12s ago 26m 57.5M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (25m) 12s ago 25m 57.6M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (25m) 12s ago 25m 50.7M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (25m) 12s ago 25m 52.0M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (24m) 12s ago 24m 54.5M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (24m) 12s ago 24m 54.6M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (24m) 12s ago 24m 52.5M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:14:47.053 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (23m) 12s ago 23m 52.9M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:14:47.054 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (51s) 12s ago 22m 52.0M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:14:47.054 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (21m) 12s ago 21m 75.9M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:14:47.054 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (21m) 12s ago 21m 77.0M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:14:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: Standby manager daemon x restarted 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: Standby manager daemon x started 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:47.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: Standby manager daemon x restarted 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/234584170' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:14:47.458 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:14:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:14:47.803 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:14:47.803 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:14:47.804 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:14:47.804 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T22:14:47.804 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:14:47.804 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "", 2024-09-13T22:14:47.804 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T22:14:47.804 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:14:47.804 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:14:48.041 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: from='client.25453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.042 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.042 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:48.042 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.042 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.042 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.042 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.042 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.043 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.043 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.043 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.043 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.043 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:48.043 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.043 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:48.044 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:48.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.047 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:48.048 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: from='client.25453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.048 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:48.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:48.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:48.053 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/713746792' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[23730]: mgrmap e42: y(active, since 65s), standbys: x 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/713746792' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:47 smithi114 ceph-mon[28528]: mgrmap e42: y(active, since 65s), standbys: x 2024-09-13T22:14:48.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: from='client.25453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:48.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:48.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-324-gecab7be5 2024-09-13T22:14:48.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:48.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:48.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:48.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:48.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:48.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/713746792' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:48.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:47 smithi121 ceph-mon[24681]: mgrmap e42: y(active, since 65s), standbys: x 2024-09-13T22:14:48.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:48.147+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:48.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:48.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:48.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:48.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:48.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:48.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:48.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:48.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:48.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:48.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='client.25465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:48.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:48.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:48.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:48.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:48.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:48.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='client.25465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:48.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:48.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:48.995 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='client.25465 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:48.996 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:48.997 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:48.997 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:49.929 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:49.929 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:49.930 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:49.931 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.932 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:49.933 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:49.934 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:49.935 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:49.936 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[23730]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[23730]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:49.937 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:49.938 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:49.939 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[28528]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[28528]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:49.940 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:49.941 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:49.941 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:49.941 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:49.941 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:49.941 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:49.791+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:49.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:49.943 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:49.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:49.945 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:49.946 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:49.947 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:49.947 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:49 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:49 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:50.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:50.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:50.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:50.066 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:49 smithi121 ceph-mon[24681]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:50.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:49 smithi121 ceph-mon[24681]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:50.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:49 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:50.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:49 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:50.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:49 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:50.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:50.999 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:51.000 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:51.001 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.002 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.003 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.004 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:51.005 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:51.006 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:51.007 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:51.008 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:51.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:51.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:51.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:51.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:51.009 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:51.105 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:51.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:51.106 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:51.107 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:51.108 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:51.109 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:51.110 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:51.111 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:51 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:52.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:51 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:52.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:14:51] "GET /metrics HTTP/1.1" 200 37998 "" "Prometheus/2.43.0" 2024-09-13T22:14:52.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:51 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:52.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:51 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:51 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:51 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:52.529 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:52.355+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:52.529 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:52.530 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:52.531 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:52.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:52.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:52.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 967 B/s rd, 0 op/s 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:53.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 967 B/s rd, 0 op/s 2024-09-13T22:14:53.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:53.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:53.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:53.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:53.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:53.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:53.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:52 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 967 B/s rd, 0 op/s 2024-09-13T22:14:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:53.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:14:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:14:53.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:52 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:14:53.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:53 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:53.941 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:53.941 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:53.941 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:53.941 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:53.941 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:53.942 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:53.943 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:53.944 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:53.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:53 smithi121 ceph-mon[24681]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:53 smithi121 ceph-mon[24681]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:53 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:53 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:53.946 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:53 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:54.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:54.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:54.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:54.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:54.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:54.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:54.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:54.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:54.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:54.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:54.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:54.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:54.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:54.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:54.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:54.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:54.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:54.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:54.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:54.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:54.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:54.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:54.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:54.081 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:54.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:54.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:54.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:54.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:54.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:54.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:54.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:54.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:54.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:54.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:54.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:54.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:54.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:54.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:54.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:54.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:54.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:54.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:54.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:54.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:54.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:54.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:54.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:54.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:54.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:54.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:54.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:54.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:54.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:54.088 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:54.089 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[23730]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[23730]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:54.090 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:54.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[28528]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[28528]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:14:54.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:53 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:14:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:14:55.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:14:55.398+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:55.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:55.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:55.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:55.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:55.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:55.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:55.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:14:55 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:55.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:14:55.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:14:55.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:55.592 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:56.481 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:56.482 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:14:56.483 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:56.484 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all crash 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all mds 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.485 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all nfs 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:14:56.486 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.487 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.487 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:14:56.487 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.487 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:14:56.487 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.487 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:56 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:14:56.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:56.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:56.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:56.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:56.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:14:56.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:56.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:56.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:14:56.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:14:56.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:14:56.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all crash 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all mds 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:14:56.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all nfs 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all crash 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all mds 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:14:56.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all nfs 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:14:56.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:14:56.589 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:56 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:14:58.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[23730]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:58.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[23730]: Upgrade: Updating grafana.a 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[23730]: Deploying daemon grafana.a on smithi121 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[28528]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[28528]: Upgrade: Updating grafana.a 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[28528]: Deploying daemon grafana.a on smithi121 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:58 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:58 smithi121 ceph-mon[24681]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:58 smithi121 ceph-mon[24681]: Upgrade: Updating grafana.a 2024-09-13T22:14:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:58 smithi121 ceph-mon[24681]: Deploying daemon grafana.a on smithi121 2024-09-13T22:14:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:58 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:58 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:58 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:14:59.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:59 smithi114 ceph-mon[23730]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:59.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:59 smithi114 ceph-mon[23730]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:59.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:14:59 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:59.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:59 smithi114 ceph-mon[28528]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:59.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:59 smithi114 ceph-mon[28528]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:59.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:14:59 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:14:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:59 smithi121 ceph-mon[24681]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:59 smithi121 ceph-mon[24681]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:14:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:14:59 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:02.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:01 smithi114 ceph-mon[23730]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-09-13T22:15:02.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:01 smithi114 ceph-mon[28528]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-09-13T22:15:02.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:15:01] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.43.0" 2024-09-13T22:15:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:01 smithi121 ceph-mon[24681]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-09-13T22:15:04.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:03 smithi114 ceph-mon[23730]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s 2024-09-13T22:15:04.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:03 smithi114 ceph-mon[28528]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s 2024-09-13T22:15:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:03 smithi121 ceph-mon[24681]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s 2024-09-13T22:15:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:05 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:05 smithi114 ceph-mon[23730]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-09-13T22:15:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:05 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:05 smithi114 ceph-mon[28528]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-09-13T22:15:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:05 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:05 smithi121 ceph-mon[24681]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-09-13T22:15:07.057 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:06 smithi121 systemd[1]: Stopping Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:15:07.332 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[55550]: t=2024-09-13T22:15:07+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-13T22:15:07.332 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 podman[77491]: 2024-09-13 22:15:07.066541631 +0000 UTC m=+0.127113659 container died 91cdcdf8ef335af58bb828aa13f145c4acfdc6b0928e752c469b3a155a66cc8e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, release=236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, build-date=2022-03-28T10:36:18.413762, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git) 2024-09-13T22:15:07.621 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 podman[77491]: 2024-09-13 22:15:07.422593263 +0000 UTC m=+0.483165282 container cleanup 91cdcdf8ef335af58bb828aa13f145c4acfdc6b0928e752c469b3a155a66cc8e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, release=236.1648460182, version=8.5, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, description=Ceph Grafana Container, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-09-13T22:15:07.621 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 bash[77491]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a 2024-09-13T22:15:08.010 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:07 smithi121 ceph-mon[24681]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:08.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 podman[77505]: 2024-09-13 22:15:07.724485701 +0000 UTC m=+0.658776521 container remove 91cdcdf8ef335af58bb828aa13f145c4acfdc6b0928e752c469b3a155a66cc8e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, architecture=x86_64, release=236.1648460182, maintainer=Paul Cuzner , io.buildah.version=1.24.2, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., version=8.5, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public) 2024-09-13T22:15:08.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T22:15:08.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 systemd[1]: Stopped Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:15:08.011 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:07 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Consumed 4.923s CPU time. 2024-09-13T22:15:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:07 smithi114 ceph-mon[23730]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:07 smithi114 ceph-mon[28528]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:08.327 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 systemd[1]: Starting Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:15:08.587 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 podman[77629]: 2024-09-13 22:15:08.230444606 +0000 UTC m=+0.016515140 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-09-13T22:15:08.587 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 podman[77629]: 2024-09-13 22:15:08.33359709 +0000 UTC m=+0.119667657 container create b45dddf54718d5f68e0ff819d4a83a4214cbe376407f533c72a9a92541b7e059 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:08.838 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 podman[77629]: 2024-09-13 22:15:08.586045298 +0000 UTC m=+0.372115824 container init b45dddf54718d5f68e0ff819d4a83a4214cbe376407f533c72a9a92541b7e059 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:08.838 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 podman[77629]: 2024-09-13 22:15:08.588993598 +0000 UTC m=+0.375064131 container start b45dddf54718d5f68e0ff819d4a83a4214cbe376407f533c72a9a92541b7e059 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:08.838 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 bash[77629]: b45dddf54718d5f68e0ff819d4a83a4214cbe376407f533c72a9a92541b7e059 2024-09-13T22:15:08.838 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 systemd[1]: Started Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:15:08.838 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741386969Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-09-13T22:15:08.838 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741552507Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T22:15:08.838 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741562616Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741565937Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741568551Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741571269Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741574238Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741576902Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741579464Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741582611Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741585276Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741587951Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741590517Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T22:15:08.839 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741593177Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T22:15:08.840 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.74159581Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T22:15:08.840 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741598309Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T22:15:08.840 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741600629Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T22:15:08.840 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741603248Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T22:15:08.840 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=settings t=2024-09-13T22:15:08.741605719Z level=info msg="App mode production" 2024-09-13T22:15:08.840 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=sqlstore t=2024-09-13T22:15:08.741661739Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=sqlstore t=2024-09-13T22:15:08.741672677Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.75292499Z level=info msg="Starting DB migrations" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.756282945Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.76763112Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.76877132Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.771691001Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.774446975Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.777248226Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.780100828Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.783014871Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.783660751Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.785757116Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.78849282Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T22:15:08.841 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.789799779Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.790348291Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.791513348Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.792685041Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.793888146Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.79422954Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.795606286Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.79673074Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.797079563Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.79838675Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.799473814Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.799900162Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.802790094Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T22:15:08.842 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.803243771Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.806048511Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.806376246Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.806794931Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.807256098Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.807334215Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.807576208Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.808081487Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.808536773Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.808797242Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.809212734Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.815537498Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.816689733Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.817007117Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T22:15:08.843 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.823218781Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.829511449Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.83069987Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.831112591Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.831566497Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.831835482Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.832288887Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.833588909Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.834354503Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.834682055Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.835027032Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.835415957Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.835885978Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.836971425Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T22:15:08.844 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.837320317Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T22:15:08.845 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.837892748Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T22:15:08.845 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.838470562Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T22:15:08.845 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.839062152Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T22:15:09.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.840753742Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.841154578Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.8415446Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.841618052Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.84185768Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.841931647Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.843648938Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.844068657Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.846768638Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.84727062Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.850026282Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T22:15:09.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.852832494Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.855184849Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.855575809Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.858411844Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.861199373Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.863753945Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.864206939Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.868065219Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.870901265Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.873585363Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.87392342Z level=info msg="Executing migration" id="add share column" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.875094563Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.875940274Z level=info msg="Executing migration" id="create file table" 2024-09-13T22:15:09.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.876914293Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.877615431Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.878566096Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.879304147Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.880288135Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.880714392Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.881731851Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.882570346Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.883415695Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.884626784Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.885611099Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.886468554Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T22:15:09.124 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.887159962Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.887691185Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.888574085Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.889914Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.891102832Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.891556598Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.904804374Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.905563856Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.906496704Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.910900214Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T22:15:09.125 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.911710637Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.912621815Z level=info msg="Executing migration" id="create folder table" 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.913347705Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.91378457Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.914546627Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.915371904Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=migrator t=2024-09-13T22:15:08.915842234Z level=info msg="migrations completed" performed=105 skipped=376 duration=159.597422ms 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=sqlstore t=2024-09-13T22:15:08.916124849Z level=info msg="Created default organization" 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=plugin.finder t=2024-09-13T22:15:08.947533092Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-13T22:15:09.126 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=secrets t=2024-09-13T22:15:08.947632366Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T22:15:09.127 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=query_data t=2024-09-13T22:15:08.951235926Z level=info msg="Query Service initialization" 2024-09-13T22:15:09.127 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=live.push_http t=2024-09-13T22:15:08.953555881Z level=info msg="Live Push Gateway initialization" 2024-09-13T22:15:09.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:09.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:09 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-09-13T22:15:09.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:09.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:09.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:09.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:09.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:09.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=infra.usagestats.collector t=2024-09-13T22:15:09.547537897Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=provisioning.datasources t=2024-09-13T22:15:09.549214578Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=provisioning.datasources t=2024-09-13T22:15:09.549223525Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=provisioning.datasources t=2024-09-13T22:15:09.54934235Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=provisioning.alerting t=2024-09-13T22:15:09.573624487Z level=info msg="starting to provision alerting" 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=provisioning.alerting t=2024-09-13T22:15:09.573640401Z level=info msg="finished to provision alerting" 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=grafanaStorageLogger t=2024-09-13T22:15:09.573838548Z level=info msg="storage starting" 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=ngalert.state.manager t=2024-09-13T22:15:09.574169356Z level=info msg="Warming state cache for startup" 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=http.server t=2024-09-13T22:15:09.576163716Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T22:15:09.829 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=ngalert.state.manager t=2024-09-13T22:15:09.58383677Z level=info msg="State cache has been initialized" states=0 duration=9.667209ms 2024-09-13T22:15:09.830 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=ticker t=2024-09-13T22:15:09.583881745Z level=info msg=starting first_tick=2024-09-13T22:15:10Z 2024-09-13T22:15:09.830 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=ngalert.multiorg.alertmanager t=2024-09-13T22:15:09.583893431Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T22:15:09.830 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=sqlstore.transactions t=2024-09-13T22:15:09.597990854Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T22:15:09.830 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=sqlstore.transactions t=2024-09-13T22:15:09.609227994Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-13T22:15:09.830 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=sqlstore.transactions t=2024-09-13T22:15:09.619997665Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-09-13T22:15:09.830 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=sqlstore.transactions t=2024-09-13T22:15:09.632639454Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=3 code="database is locked" 2024-09-13T22:15:10.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:10.260+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:10.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:10.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:10.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:10.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:10.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:10.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:10.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:10.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:10.991 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:10 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:11.022 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:11.023 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:11.024 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:11.024 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:11.024 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:11.024 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:10 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:11.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:11 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:11.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:11.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:11.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:11.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:11.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:11 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:11 smithi121 ceph-mon[24681]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:11 smithi121 ceph-mon[24681]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:11 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:11 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:11.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:11 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:12.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:12.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:12.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:12.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:12.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:15:11] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.43.0" 2024-09-13T22:15:12.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:12.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:12.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:12.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:12.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:12.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:12.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:12.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:12.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:12.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:12.082 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[28528]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:12.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:12.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[23730]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[28528]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:12.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:12.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:12.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:12.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[23730]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:12.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:12.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:12.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:11 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:12.521+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.814 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.815 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:12.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:12.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:12.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:12.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:12.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:12.816 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:12.817 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:12.818 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:12.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:12.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:12.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:12.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:12.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:12.819 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:13.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:13.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:12 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:13.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:13.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:13.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:13.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:13.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:13.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:12 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:13.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:13.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:13.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:13.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:13.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:13.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:13 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:14.068 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:14.069 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:14.070 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.071 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.072 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:14.073 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:14.074 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:14.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:14.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:13 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:15.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:15 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:15.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:15 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:15 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:15 smithi121 ceph-mon[24681]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:15.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:15 smithi121 ceph-mon[24681]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:15:15.522 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:15 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[23730]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[23730]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:15.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[28528]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:15.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[28528]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:15:15.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:16.057+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:16.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:16.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:16.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:16.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:16.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:16.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:16.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:16.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:16.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:16.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:16.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:16.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:16.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:16.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:16.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:16.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:16.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:16.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:16.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:16.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:16.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:16.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:16.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:16 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:17.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:16 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:18.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:18.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:18.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:18.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:18.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:17.698+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:18.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.086 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:18.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.091 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.091 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:18.091 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.091 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.092 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:18.093 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:18.094 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:18.095 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:18.096 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:18.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:18.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:18.097 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:17 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:18.243 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:15:18.923 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:15:18.923 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 4s ago 27m - - 2024-09-13T22:15:18.923 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 0s ago 27m - - 2024-09-13T22:15:18.923 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (14m) 4s ago 22m 25.5M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (13m) 0s ago 22m 57.1M - 8.3.5 dad864ee21e9 91cdcdf8ef33 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 4s ago 21m 46.5M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (37s) 0s ago 27m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (10m) 4s ago 28m 513M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (28m) 4s ago 28m 84.4M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (27m) 0s ago 27m 66.9M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (27m) 4s ago 27m 62.5M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (14m) 4s ago 23m 14.8M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (13m) 0s ago 23m 15.0M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (26m) 4s ago 26m 57.4M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (26m) 4s ago 26m 57.5M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (26m) 4s ago 26m 50.7M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:15:18.924 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (25m) 4s ago 25m 51.9M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:15:18.925 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (25m) 0s ago 25m 54.5M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:15:18.925 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (24m) 0s ago 24m 54.5M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:15:18.925 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (24m) 0s ago 24m 52.5M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:15:18.925 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (24m) 0s ago 24m 53.0M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:15:18.925 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (83s) 0s ago 23m 52.8M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:15:18.925 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (21m) 4s ago 21m 76.0M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:15:18.925 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (21m) 0s ago 21m 77.1M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:15:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 0 op/s 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 0 op/s 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:19.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.087 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 0 op/s 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 0 op/s 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.088 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:18 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:19.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:19.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:19.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 0 op/s 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 989 B/s rd, 0 op/s 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:19.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:19.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:19.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:18 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:19.329 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:15:19.329 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:15:19.329 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:15:19.329 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:15:19.329 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:15:19.330 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:15:19.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:19.370+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:19.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:19.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:19.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:19.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:19.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:19.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:19.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:19.675 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [ 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr" 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: ], 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:15:19.676 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:15:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='client.25489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2113131438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='client.25489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2113131438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:19 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='client.25489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2113131438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:19 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:21.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:20 smithi121 ceph-mon[24681]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:20 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:21.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:21.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:21.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:20 smithi121 ceph-mon[24681]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:20 smithi114 ceph-mon[23730]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:20 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:21.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:21.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:21.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:21.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:21.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:21.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:21.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:21.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:21.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:21.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:21.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:21.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:21.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:21.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:21.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:21.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:21.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:21.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:21.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:21.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:21.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:21.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:21.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:21.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:21.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:21.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:21.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:21.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:21.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:21.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:21.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:21.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:21.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:21.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:21.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:21.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:21.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:21.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:21.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:21.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:21.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:21.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:21.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:21.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:20 smithi114 ceph-mon[28528]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:21.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:20 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:21.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:21.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:21.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:21.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:21.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:21.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:21.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:21.339 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:21.339 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:21.339 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:21.339 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:21.339 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:21.339 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:21.339 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:21.340 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:21.340 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:21.340 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:21.340 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:21.340 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:21.341 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:21.341 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:21.341 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:21.341 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:21.341 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:21.341 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:21.342 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.342 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.342 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:21.342 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:21.342 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:21.342 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:21.343 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:20 smithi114 ceph-mon[28528]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:21.344 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:20 smithi114 ceph-mon[23730]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:22.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:15:21] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[23730]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[28528]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:21 smithi121 ceph-mon[24681]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:22.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:22 smithi121 ceph-mon[24681]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s 2024-09-13T22:15:23.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:22 smithi121 ceph-mon[24681]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:15:23.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:22 smithi114 ceph-mon[23730]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s 2024-09-13T22:15:23.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:22 smithi114 ceph-mon[23730]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:15:23.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:22 smithi114 ceph-mon[28528]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s 2024-09-13T22:15:23.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:22 smithi114 ceph-mon[28528]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:15:25.124 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:24 smithi121 ceph-mon[24681]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-09-13T22:15:25.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:24 smithi114 ceph-mon[23730]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-09-13T22:15:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:24 smithi114 ceph-mon[28528]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-09-13T22:15:25.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:25 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:26.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:25 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:26.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:25 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:27.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:26 smithi121 ceph-mon[24681]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:15:27.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:26 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[23730]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:15:27.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[28528]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:15:27.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:27.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:27 smithi121 ceph-mon[24681]: Upgrade: Updating grafana.a 2024-09-13T22:15:28.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:27 smithi121 ceph-mon[24681]: Deploying daemon grafana.a on smithi121 2024-09-13T22:15:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:27 smithi121 ceph-mon[24681]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:28.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:27 smithi121 ceph-mon[24681]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-09-13T22:15:28.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[23730]: Upgrade: Updating grafana.a 2024-09-13T22:15:28.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[23730]: Deploying daemon grafana.a on smithi121 2024-09-13T22:15:28.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[23730]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:28.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[23730]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-09-13T22:15:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[28528]: Upgrade: Updating grafana.a 2024-09-13T22:15:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[28528]: Deploying daemon grafana.a on smithi121 2024-09-13T22:15:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[28528]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:27 smithi114 ceph-mon[28528]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-09-13T22:15:28.958 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:28 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:29.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:30.119 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:29 smithi121 ceph-mon[24681]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 696 B/s rd, 0 op/s 2024-09-13T22:15:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:29 smithi114 ceph-mon[23730]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 696 B/s rd, 0 op/s 2024-09-13T22:15:30.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:29 smithi114 ceph-mon[28528]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 696 B/s rd, 0 op/s 2024-09-13T22:15:31.521 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:31 smithi121 systemd[1]: Stopping Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:15:31.522 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=server t=2024-09-13T22:15:31.510524962Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-13T22:15:31.522 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=ticker t=2024-09-13T22:15:31.510809236Z level=info msg=stopped last_tick=2024-09-13T22:15:30Z 2024-09-13T22:15:31.522 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[77645]: logger=tracing t=2024-09-13T22:15:31.510830333Z level=info msg="Closing tracing" 2024-09-13T22:15:31.820 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:31 smithi121 podman[79352]: 2024-09-13 22:15:31.520778666 +0000 UTC m=+0.109516623 container died b45dddf54718d5f68e0ff819d4a83a4214cbe376407f533c72a9a92541b7e059 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:31 smithi114 ceph-mon[23730]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:15:31] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.43.0" 2024-09-13T22:15:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:31 smithi114 ceph-mon[28528]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:32.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:31 smithi121 ceph-mon[24681]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:32.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:31 smithi121 podman[79352]: 2024-09-13 22:15:31.902232653 +0000 UTC m=+0.490970601 container cleanup b45dddf54718d5f68e0ff819d4a83a4214cbe376407f533c72a9a92541b7e059 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:32.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:31 smithi121 bash[79352]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a 2024-09-13T22:15:32.422 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 podman[79365]: 2024-09-13 22:15:32.161695298 +0000 UTC m=+0.638050542 container remove b45dddf54718d5f68e0ff819d4a83a4214cbe376407f533c72a9a92541b7e059 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:32.422 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T22:15:32.422 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 systemd[1]: Stopped Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:15:32.422 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Consumed 2.535s CPU time. 2024-09-13T22:15:32.731 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 systemd[1]: Starting Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:15:33.007 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 podman[79491]: 2024-09-13 22:15:32.633844182 +0000 UTC m=+0.016831019 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-09-13T22:15:33.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 podman[79491]: 2024-09-13 22:15:32.755176568 +0000 UTC m=+0.138163391 container create 6ee3553a890cd6f14da8c1ebbee2f430d96849a5a54366051c38beb73dc5d50e (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:33.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 podman[79491]: 2024-09-13 22:15:32.949042065 +0000 UTC m=+0.332028889 container init 6ee3553a890cd6f14da8c1ebbee2f430d96849a5a54366051c38beb73dc5d50e (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:33.008 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:32 smithi121 podman[79491]: 2024-09-13 22:15:32.952716023 +0000 UTC m=+0.335702855 container start 6ee3553a890cd6f14da8c1ebbee2f430d96849a5a54366051c38beb73dc5d50e (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 bash[79491]: 6ee3553a890cd6f14da8c1ebbee2f430d96849a5a54366051c38beb73dc5d50e 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 systemd[1]: Started Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102272985Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102448225Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102459471Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102463093Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102466085Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102468903Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102471602Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102474862Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102477594Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-13T22:15:33.372 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102480861Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102483736Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102486385Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102488971Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.1024917Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102494395Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102496946Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102499895Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102502651Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=settings t=2024-09-13T22:15:33.102505317Z level=info msg="App mode production" 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=sqlstore t=2024-09-13T22:15:33.102550674Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=sqlstore t=2024-09-13T22:15:33.102562706Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=migrator t=2024-09-13T22:15:33.116707457Z level=info msg="Starting DB migrations" 2024-09-13T22:15:33.373 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=migrator t=2024-09-13T22:15:33.121262638Z level=info msg="migrations completed" performed=0 skipped=481 duration=411.342µs 2024-09-13T22:15:33.374 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=sqlstore t=2024-09-13T22:15:33.121610218Z level=info msg="Created default organization" 2024-09-13T22:15:33.374 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=plugin.finder t=2024-09-13T22:15:33.154327496Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-13T22:15:33.374 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=secrets t=2024-09-13T22:15:33.154430867Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T22:15:33.374 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=query_data t=2024-09-13T22:15:33.157833953Z level=info msg="Query Service initialization" 2024-09-13T22:15:33.374 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=live.push_http t=2024-09-13T22:15:33.160452713Z level=info msg="Live Push Gateway initialization" 2024-09-13T22:15:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:34.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:34.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=infra.usagestats.collector t=2024-09-13T22:15:33.753320706Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T22:15:34.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=provisioning.datasources t=2024-09-13T22:15:33.755268243Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T22:15:34.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=provisioning.datasources t=2024-09-13T22:15:33.755276234Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T22:15:34.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=provisioning.datasources t=2024-09-13T22:15:33.755424282Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T22:15:34.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=provisioning.alerting t=2024-09-13T22:15:33.781849889Z level=info msg="starting to provision alerting" 2024-09-13T22:15:34.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=provisioning.alerting t=2024-09-13T22:15:33.781860001Z level=info msg="finished to provision alerting" 2024-09-13T22:15:34.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=ngalert.state.manager t=2024-09-13T22:15:33.781967248Z level=info msg="Warming state cache for startup" 2024-09-13T22:15:34.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=grafanaStorageLogger t=2024-09-13T22:15:33.783126716Z level=info msg="storage starting" 2024-09-13T22:15:34.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=http.server t=2024-09-13T22:15:33.790094369Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T22:15:34.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=ngalert.state.manager t=2024-09-13T22:15:33.792545205Z level=info msg="State cache has been initialized" states=0 duration=10.576751ms 2024-09-13T22:15:34.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=ticker t=2024-09-13T22:15:33.792600545Z level=info msg=starting first_tick=2024-09-13T22:15:40Z 2024-09-13T22:15:34.123 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:15:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=ngalert.multiorg.alertmanager t=2024-09-13T22:15:33.792605148Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T22:15:34.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:34.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:34.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:34.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:33 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:34.960 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 669 B/s rd, 0 op/s 2024-09-13T22:15:34.960 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 813 B/s rd, 0 op/s 2024-09-13T22:15:34.960 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 669 B/s rd, 0 op/s 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 813 B/s rd, 0 op/s 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.961 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:34.962 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 669 B/s rd, 0 op/s 2024-09-13T22:15:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 813 B/s rd, 0 op/s 2024-09-13T22:15:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:35.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:35.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:35.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:35.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:35.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:34 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:36.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:36.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:36.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:15:36.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:36.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:36.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:36.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:36.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:36.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:36.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:35.780+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:36.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:36.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:36.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:36.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:36.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:36.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:36.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:36.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:36.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:36.081 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:15:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:15:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:36.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:35 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:37.070 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:36 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.070 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:37.070 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.071 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:37.072 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.073 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.074 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.075 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:36 smithi121 ceph-mon[24681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:37.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:36 smithi121 ceph-mon[24681]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-09-13T22:15:37.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:36 smithi121 ceph-mon[24681]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:37.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:36 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:37.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:36 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:37.075 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:36 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:37.322 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:37.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:37.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[23730]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:37.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[23730]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[28528]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[28528]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[23730]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[28528]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:37.332 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:37.333 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:36 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:37.508+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:37.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:37.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:37.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.071 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:38.071 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:38.071 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:38.071 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:38.071 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:38.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:38.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:38.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:38.072 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:37 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:38.165 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:38.166 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:38.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:37 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.991 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:38.992 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:38.993 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:38.994 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:38.995 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:38.996 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:38.997 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[23730]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:38.998 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:38.999 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[28528]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:39.000 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[23730]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:39.001 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:39.001 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:39.001 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:39.001 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[28528]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:39.001 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:39.001 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:39.001 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:38 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:39.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:38 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:39.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:39.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:39.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:39.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:38 smithi121 ceph-mon[24681]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:39.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:38 smithi121 ceph-mon[24681]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:39.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:38 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:39.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:38 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:39.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:38 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:39.092+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:39.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:39.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:39.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:39.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:39.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:40.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:40.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:39 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:40.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:40.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:40.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:40.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:40.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:39 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:40.655+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:40.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:40.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:40.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:40.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:40.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:40.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:40.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:40.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:40.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.014 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:41.014 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:41.015 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:41.016 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:41.017 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:41.018 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:41.019 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:41.020 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:41.020 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:41.020 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:41.020 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:40 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:41.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:41.157 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:41.157 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:41.158 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:41.159 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:41.160 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:41.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:41.160 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:41.160 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:41.160 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:41.160 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:41.160 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:41.161 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:41.162 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:41.163 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:41.164 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.165 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.165 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:41.165 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:41.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:41.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:41.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:41.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:41.166 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:41.167 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:41.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:41.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:40 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:42.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:15:41] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.43.0" 2024-09-13T22:15:42.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:42.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:42.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:42.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[23730]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:42.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[28528]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[23730]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:42.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:42.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:42.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[28528]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:15:42.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:42.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:42.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:41 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:42.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:41 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:42.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:42.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:41 smithi121 ceph-mon[24681]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:41 smithi121 ceph-mon[24681]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:15:42.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:41 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:42.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:41 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:42.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:41 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:42.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:42.226+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:42.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:42.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:42.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:42.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:42.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:42.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:42.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:42.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:42.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:43.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:43.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:43.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:43.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:42 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:43.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:43.125 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:42 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:43.947 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:43.947 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.947 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.947 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.948 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:43.949 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.950 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.951 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.952 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.953 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:43.954 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.955 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.956 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:43.957 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:43.958 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:43.959 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:43.778+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.959 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:43.959 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.959 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.959 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.959 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.960 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:43.961 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:43.962 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:43.963 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:44.038 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:44.039 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:44.040 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:44.041 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:44.042 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:44.043 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:43 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:45.109 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:44 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:45.109 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:45.109 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:44 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:45.109 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:44 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:45.109 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:44 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:45.160 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:45.161 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:44 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:45.208+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:45.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:45.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:45.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:45.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:45.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:45.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:45.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:45.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:45.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.917 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:45.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:45.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:45.917 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:45.918 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:45.919 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.920 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:45.921 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:45.922 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:45 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:46.058 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:46.059 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.060 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:46.063 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:46.066 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:46.067 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:46.068 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:45 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.074 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:47.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:47.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:47.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:47.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:47.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:47.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:47.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:47.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:47.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:47.086 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:47.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:47.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:47.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:47.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:47.087 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:47.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:46.775+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.087 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.088 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:47.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.092 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.092 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.092 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:47.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:47.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:47.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:47.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:47.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:46 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[28528]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[28528]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.216 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:48.217 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:48.217 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.217 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.217 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[23730]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:48.217 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[23730]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:48.217 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.341 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:47 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:48.341 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:47 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:48.341 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.341 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.342 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:47 smithi121 ceph-mon[24681]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:48.342 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:47 smithi121 ceph-mon[24681]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:48.342 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:47 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:48.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:48.328+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:48.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:48.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:48.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:48.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:48.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:48.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:48.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:48.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:48.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.057 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:49.057 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:49.058 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:49.059 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:49.059 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:49.059 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:49.059 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:49.059 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:49.060 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.060 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.060 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.060 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:49.060 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:49.060 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:49.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:49.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:49.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:49.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:49.061 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:49.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:49.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:49.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:49.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:49.062 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.063 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:49.064 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.065 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: Upgrade: Finalizing container_image settings 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:15:49.066 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:15:49.067 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: Upgrade: Complete! 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:49.068 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.069 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.070 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:48 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.162 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.163 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:49.163 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:49.163 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:49.163 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:49.163 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:49.163 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:49.164 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:49.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:49.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:49.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:49.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:49.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:49.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:49.165 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:49.166 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:49.167 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:49.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:49.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:49.170 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:49.171 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:49.172 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.173 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.174 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: Upgrade: Finalizing container_image settings 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:15:49.175 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.176 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: Upgrade: Finalizing container_image settings 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:15:49.177 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:15:49.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:15:49.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:15:49.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:15:49.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:15:49.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.179 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.180 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.180 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: Upgrade: Complete! 2024-09-13T22:15:49.180 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:15:49.180 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:15:49.180 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:49.180 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.181 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.182 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:15:49.182 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.182 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:15:49.183 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: Upgrade: Complete! 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:49.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.287 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.287 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.288 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:50 smithi121 ceph-mon[24681]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:50.288 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:50 smithi121 ceph-mon[24681]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:50.288 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:50 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[23730]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:50.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[23730]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:50.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[28528]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[28528]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:15:50.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:50.003+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:50.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:50.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:50.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:50.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:50.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:50.668 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-13T22:15:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:51.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:51.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:51.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:51.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:51.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:51.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:51.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:51.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:51.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:51.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:51.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:51.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:51.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:51.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:51.335 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:51.336 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T22:15:51.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T22:15:51.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:51.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:51.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:51.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:51.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:51.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:51.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:51.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:51.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:15:51.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T22:15:51.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-13T22:15:51.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:51.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:15:51.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:15:51.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:15:51.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:51 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:15:52.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:15:51] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.43.0" 2024-09-13T22:15:53.211 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:15:53.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-mon[23730]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:53.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-mon[28528]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:53 smithi121 ceph-mon[24681]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:15:53.791 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:15:53.612+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:53.791 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:53.791 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:53.792 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:53.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:53.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:53.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:53.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:53.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:53.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:53.793 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:53.794 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:53.795 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:53.796 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:53.796 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:53.796 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:53.796 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:53.796 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:53.796 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:15:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:54 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1414474512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:54.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:54 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:54.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:54.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:54.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:54.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:15:54.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.409 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:54 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1414474512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:54.409 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:54 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:54.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:54.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:54.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:54.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:54.410 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:54.411 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:54.412 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:54.413 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.414 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:54 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1414474512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:54 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:54.415 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.416 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:15:54.417 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:15:54.418 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:15:54.419 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:15:54.420 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:15:54.420 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:15:54.420 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:15:54.578 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-09-13T22:15:55.955 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:55.955 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[23730]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-09-13T22:15:55.955 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:55.955 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[23730]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[23730]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[28528]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[28528]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[28528]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:55.956 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:55 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:55 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:15:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:55 smithi121 ceph-mon[24681]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-09-13T22:15:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:56.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:55 smithi121 ceph-mon[24681]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:56.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:55 smithi121 ceph-mon[24681]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:15:56.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:55 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:15:57.362 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T22:15:58.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:57 smithi114 ceph-mon[23730]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 772 B/s rd, 0 op/s 2024-09-13T22:15:58.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:57 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1189035891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:58.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:57 smithi114 ceph-mon[28528]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 772 B/s rd, 0 op/s 2024-09-13T22:15:58.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:57 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1189035891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:57 smithi121 ceph-mon[24681]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 772 B/s rd, 0 op/s 2024-09-13T22:15:58.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:57 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1189035891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:15:58.466 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-09-13T22:15:59.023 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:58 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:59.023 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:58 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:15:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:58 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:00.037 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:15:59 smithi114 ceph-mon[23730]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 593 B/s rd, 0 op/s 2024-09-13T22:16:00.037 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:15:59 smithi114 ceph-mon[28528]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 593 B/s rd, 0 op/s 2024-09-13T22:16:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:15:59 smithi121 ceph-mon[24681]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 593 B/s rd, 0 op/s 2024-09-13T22:16:00.594 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:16:00.951 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:00 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3151022406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:00.952 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:00 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3151022406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:00 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/3151022406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:01.244 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-09-13T22:16:01.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:16:01] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-09-13T22:16:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:01 smithi121 ceph-mon[24681]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:02.192 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:01 smithi114 ceph-mon[23730]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:02.192 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:01 smithi114 ceph-mon[28528]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:03.943 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:03 smithi114 ceph-mon[23730]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:03.943 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:03 smithi114 ceph-mon[28528]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:03 smithi121 ceph-mon[24681]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:04.603 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:16:05.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:04 smithi121 ceph-mon[24681]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:05.189 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:04 smithi114 ceph-mon[23730]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:05.189 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:04 smithi114 ceph-mon[28528]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:05.211 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T22:16:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:05 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:06.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:05 smithi114 ceph-mon[23730]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:16:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:05 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:06.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:05 smithi114 ceph-mon[28528]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:16:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:05 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:05 smithi121 ceph-mon[24681]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:16:07.210 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:16:07.210 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": null, 2024-09-13T22:16:07.210 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": false, 2024-09-13T22:16:07.211 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "", 2024-09-13T22:16:07.211 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:16:07.211 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": null, 2024-09-13T22:16:07.211 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "", 2024-09-13T22:16:07.211 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:16:07.211 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:16:07.783 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T22:16:08.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:07 smithi114 ceph-mon[23730]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:16:08.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:07 smithi114 ceph-mon[28528]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:16:08.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:07 smithi121 ceph-mon[24681]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:16:09.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:08 smithi114 ceph-mon[28528]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:09.017 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:08 smithi114 ceph-mon[23730]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:08 smithi121 ceph-mon[24681]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:09.810 INFO:teuthology.orchestra.run.smithi114.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:16:09.810 INFO:teuthology.orchestra.run.smithi114.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:16:09.810 INFO:teuthology.orchestra.run.smithi114.stdout: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:16:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[23730]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:16:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2085212773' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:16:10.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[28528]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:16:10.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:09 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2085212773' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:16:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:09 smithi121 ceph-mon[24681]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:16:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:09 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:10.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:09 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/2085212773' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:16:10.482 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-09-13T22:16:11.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:10 smithi114 ceph-mon[28528]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-09-13T22:16:11.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:10 smithi114 ceph-mon[28528]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-13T22:16:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:10 smithi114 ceph-mon[23730]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-09-13T22:16:11.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:10 smithi114 ceph-mon[23730]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-13T22:16:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:10 smithi121 ceph-mon[24681]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-09-13T22:16:11.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:10 smithi121 ceph-mon[24681]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-13T22:16:12.021 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:16:11] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-09-13T22:16:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:12 smithi121 ceph-mon[24681]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:13.195 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:12 smithi114 ceph-mon[23730]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:13.195 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:12 smithi114 ceph-mon[28528]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:14.192 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:13 smithi114 ceph-mon[23730]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:14.192 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:13 smithi114 ceph-mon[23730]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi121", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:14.192 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:13 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:14.193 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:13 smithi114 ceph-mon[28528]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:14.193 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:13 smithi114 ceph-mon[28528]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi121", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:14.193 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:13 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:14.228 INFO:teuthology.orchestra.run.smithi114.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:13 smithi121 ceph-mon[24681]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:13 smithi121 ceph-mon[24681]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi121", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:14.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:13 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:15.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-13T22:16:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:15.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-13T22:16:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:15.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:15.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:15.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:15.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-13T22:16:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:15.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:15.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:15.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:14 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:15.432 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T22:16:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:16.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[23730]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:15 smithi114 ceph-mon[28528]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:15 smithi121 ceph-mon[24681]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:16:17.859 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:17.670+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:17.860 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:17.861 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:17.862 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:17.863 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:17.864 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:18.235 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-mon[23730]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:16:18.236 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:17 smithi114 ceph-mon[28528]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:16:18.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:17 smithi121 ceph-mon[24681]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:16:18.709 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:16:19.171 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:18 smithi121 ceph-mon[24681]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:19.171 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:19.171 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:19.171 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:19.171 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:19.171 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:19.172 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:19.173 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:19.174 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:19.175 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.176 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.176 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:18 smithi121 ceph-mon[24681]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:19.320 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:18 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:19.320 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:19.321 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:19.322 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:19.323 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:19.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:18 smithi114 ceph-mon[23730]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:18 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:19.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:19.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:19.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:18 smithi114 ceph-mon[28528]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:19.434 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:16:19.434 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 4s ago 28m - - 2024-09-13T22:16:19.434 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 10s ago 28m - - 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (15m) 4s ago 23m 25.8M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (46s) 10s ago 23m 76.1M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 4s ago 22m 51.1M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (98s) 10s ago 28m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (11m) 4s ago 29m 518M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (29m) 4s ago 29m 92.7M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (28m) 10s ago 28m 70.1M 2048M 17.2.0 e1d6a67b021e 7b73e0e68a9e 2024-09-13T22:16:19.435 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (28m) 4s ago 28m 61.1M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (15m) 4s ago 24m 14.7M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (14m) 10s ago 24m 15.0M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (27m) 4s ago 27m 57.5M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (27m) 4s ago 27m 57.6M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (27m) 4s ago 27m 50.7M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (26m) 4s ago 26m 51.9M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (26m) 10s ago 26m 54.5M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (25m) 10s ago 25m 54.5M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (25m) 10s ago 25m 52.5M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:16:19.436 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (25m) 10s ago 25m 53.0M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:16:19.437 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (2m) 10s ago 24m 58.8M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:16:19.437 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (22m) 4s ago 22m 76.2M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:16:19.437 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (22m) 10s ago 22m 77.3M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:16:19.843 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:16:19.844 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:16:19.845 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:16:19.845 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:16:19.845 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:16:19.845 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:16:19.845 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:16:19.845 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:16:19.845 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:16:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[23730]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[23730]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:16:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[23730]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[23730]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/1836346699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[28528]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[28528]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:16:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[28528]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[28528]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:19 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/1836346699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi121", 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "", 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 image", 2024-09-13T22:16:20.183 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:16:20.184 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:16:20.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:19 smithi121 ceph-mon[24681]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:19 smithi121 ceph-mon[24681]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:16:20.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:19 smithi121 ceph-mon[24681]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:19 smithi121 ceph-mon[24681]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:20.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:19 smithi121 ceph-mon[24681]: from='client.? 172.21.15.114:0/1836346699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:21.656 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:21.656 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:21.656 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:16:21.656 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:16:21.656 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:21.657 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:21.657 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: Upgrade: Setting container_image for all mgr 2024-09-13T22:16:21.657 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:21.657 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:16:21.657 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T22:16:21.657 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: Upgrade: It appears safe to stop mon.b 2024-09-13T22:16:21.657 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:21 smithi121 ceph-mon[24681]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all mgr 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:21.725 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: Upgrade: It appears safe to stop mon.b 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[23730]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all mgr 2024-09-13T22:16:21.726 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:21.727 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:16:21.727 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T22:16:21.727 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: Upgrade: It appears safe to stop mon.b 2024-09-13T22:16:21.727 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-mon[28528]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-09-13T22:16:22.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:16:21] "GET /metrics HTTP/1.1" 200 38009 "" "Prometheus/2.43.0" 2024-09-13T22:16:23.046 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:22 smithi121 ceph-mon[24681]: Upgrade: Updating mon.b 2024-09-13T22:16:23.046 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:23.046 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:16:23.047 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:16:23.047 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:22 smithi121 ceph-mon[24681]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:23.047 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:22 smithi121 ceph-mon[24681]: Deploying daemon mon.b on smithi121 2024-09-13T22:16:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[28528]: Upgrade: Updating mon.b 2024-09-13T22:16:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:23.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[28528]: Deploying daemon mon.b on smithi121 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[23730]: Upgrade: Updating mon.b 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:22 smithi114 ceph-mon[23730]: Deploying daemon mon.b on smithi121 2024-09-13T22:16:23.577 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:23 smithi121 systemd[1]: Stopping Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:16:23.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b[24664]: 2024-09-13T22:16:23.574+0000 7f0b9123e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:16:23.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b[24664]: 2024-09-13T22:16:23.574+0000 7f0b9123e700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T22:16:23.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:23 smithi121 podman[81804]: 2024-09-13 22:16:23.698203974 +0000 UTC m=+0.229516303 container died 7b73e0e68a9e27d584c56c5128e31a51c24bd6124024eece248b69b195c046c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_CLEAN=True, vcs-type=git, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T22:16:24.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:24 smithi121 podman[81804]: 2024-09-13 22:16:24.049963294 +0000 UTC m=+0.581275632 container cleanup 7b73e0e68a9e27d584c56c5128e31a51c24bd6124024eece248b69b195c046c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, release=754, name=centos-stream, version=8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64) 2024-09-13T22:16:24.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:24 smithi121 bash[81804]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b 2024-09-13T22:16:24.716 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:24 smithi121 podman[81816]: 2024-09-13 22:16:24.46029984 +0000 UTC m=+0.756293620 container remove 7b73e0e68a9e27d584c56c5128e31a51c24bd6124024eece248b69b195c046c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.component=centos-stream-container, version=8, ceph=True, RELEASE=HEAD, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream) 2024-09-13T22:16:24.716 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:24 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T22:16:24.717 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:24 smithi121 systemd[1]: Stopped Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:16:24.717 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:24 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.b.service: Consumed 19.095s CPU time. 2024-09-13T22:16:25.020 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:24 smithi121 systemd[1]: Starting Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:16:25.272 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 podman[81946]: 2024-09-13 22:16:25.019694486 +0000 UTC m=+0.109403747 container create 89a5df53ca799b21800e1cdf14138b1c4c905ffc524f71d79c9d285e77549099 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T22:16:25.272 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 podman[81946]: 2024-09-13 22:16:24.925889722 +0000 UTC m=+0.015598945 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:25.273 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 podman[81946]: 2024-09-13 22:16:25.247280883 +0000 UTC m=+0.336990110 container init 89a5df53ca799b21800e1cdf14138b1c4c905ffc524f71d79c9d285e77549099 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-13T22:16:25.273 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 podman[81946]: 2024-09-13 22:16:25.250409299 +0000 UTC m=+0.340118519 container start 89a5df53ca799b21800e1cdf14138b1c4c905ffc524f71d79c9d285e77549099 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:16:25.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: pidfile_write: ignore empty --pid-file 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: load: jerasure load: lrc 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: RocksDB version: 7.9.2 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Git sha 0 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: DB SUMMARY 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: DB Session ID: AIIFR995L272M018KAQS 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:16:25.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3791 Bytes 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000069.sst 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000067.log size: 455554 ; 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.env: 0x561774f05940 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.info_log: 0x5617764116e0 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:16:25.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.statistics: (nil) 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.use_fsync: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:16:25.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.db_log_dir: 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.wal_dir: 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.write_buffer_manager: 0x5617764208c0 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:16:25.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.unordered_write: 0 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.row_cache: None 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.wal_filter: None 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:16:25.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.wal_compression: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:16:25.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:16:25.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_open_files: -1 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Compression algorithms supported: 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kZSTD supported: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:16:25.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.merge_operator: 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_filter: None 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:16:25.631 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561776411320) 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_type: 0 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: data_block_index_type: 0 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_shortening: 1 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: checksum: 4 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: no_block_cache: 0 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache: 0x561776405350 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_options: 2024-09-13T22:16:25.632 INFO:journalctl@ceph.mon.b.smithi121.stdout: capacity : 536870912 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: num_shard_bits : 4 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: strict_capacity_limit : 0 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_compressed: (nil) 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: persistent_cache: (nil) 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_size: 4096 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_size_deviation: 10 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_restart_interval: 16 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_block_restart_interval: 1 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: metadata_block_size: 4096 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: partition_filters: 0 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: use_delta_encoding: 1 2024-09-13T22:16:25.633 INFO:journalctl@ceph.mon.b.smithi121.stdout: filter_policy: bloomfilter 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: whole_key_filtering: 1 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: verify_compression: 0 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: format_version: 5 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: enable_index_compression: 1 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_align: 0 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: max_auto_readahead_size: 262144 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: prepopulate_block_cache: 0 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: initial_auto_readahead_size: 8192 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression: NoCompression 2024-09-13T22:16:25.634 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.num_levels: 7 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:16:25.635 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:16:25.636 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:16:25.637 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:16:25.638 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:16:25.639 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.ttl: 2592000 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T22:16:25.640 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 69.sst 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 71, last_sequence is 37225, log_number is 67,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 67 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 61c5307a-d4a1-4f7e-ad8a-c321313872c7 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265785277330, "job": 1, "event": "recovery_started", "wal_files": [67]} 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #67 mode 2 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265785279267, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 72, "file_size": 299564, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 37222, "largest_seqno": 37516, "table_properties": {"data_size": 297301, "index_size": 839, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3485, "raw_average_key_size": 25, "raw_value_size": 294221, "raw_average_value_size": 2132, "num_data_blocks": 37, "num_entries": 138, "num_filter_entries": 138, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726265785, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "61c5307a-d4a1-4f7e-ad8a-c321313872c7", "db_session_id": "AIIFR995L272M018KAQS", "orig_file_number": 72, "seqno_to_time_mapping": "N/A"}} 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265785279363, "job": 1, "event": "recovery_finished"} 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/version_set.cc:5047] Creating manifest 74 2024-09-13T22:16:25.641 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000067.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561776432e00 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: DB pointer 0x56177652e000 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** DB Stats ** 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:16:25.642 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** Compaction Stats [default] ** 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: L0 1/0 292.54 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 248.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: L6 1/0 11.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: Sum 2/0 11.30 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 248.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 248.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** Compaction Stats [default] ** 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 248.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:25.643 INFO:journalctl@ceph.mon.b.smithi121.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative compaction: 0.00 GB write, 68.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval compaction: 0.00 GB write, 68.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: Block cache BinnedLRUCache@0x561776405350#2 capacity: 512.00 MB usage: 1.39 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,0.95 KB,0.000181794%) Misc(1,0.00 KB,0%) 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:25.644 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: starting mon.b rank 2 at public addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] at bind addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???) e3 preinit fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???).mds e1 new map 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???).mds e1 print_map 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout: e1 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout: legacy client fscid: -1 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout: No filesystems configured 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T22:16:25.645 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:16:25.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:16:25.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:16:25.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 ceph-mon[81965]: mon.b@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-09-13T22:16:25.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 bash[81946]: 89a5df53ca799b21800e1cdf14138b1c4c905ffc524f71d79c9d285e77549099 2024-09-13T22:16:25.646 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:25 smithi121 systemd[1]: Started Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:16:26.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: mon.b calling monitor election 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: mon.c calling monitor election 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: mon.a calling monitor election 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: fsmap 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:16:26.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:26.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: mon.b calling monitor election 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: mon.c calling monitor election 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: mon.a calling monitor election 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: fsmap 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:16:26.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:26.828 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:26 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: mon.b calling monitor election 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: mon.c calling monitor election 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: mon.a calling monitor election 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T22:16:26.842 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: fsmap 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:26.843 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:26.844 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:26.844 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:26.844 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:26.844 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:26 smithi121 ceph-mon[81965]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:27.038+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:27.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:27.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:27.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:27 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:27 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:27 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:27 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:27 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:27 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:27.778 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:27 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:27.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:27.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:27.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:27.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:27.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:27 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:28.688 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:28 smithi121 ceph-mon[81965]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.688 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.689 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:28.690 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.691 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.692 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:28 smithi121 ceph-mon[81965]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:28 smithi121 ceph-mon[81965]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 815 B/s rd, 0 op/s 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:28 smithi121 ceph-mon[81965]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:28 smithi121 ceph-mon[81965]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:28 smithi121 ceph-mon[81965]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:28.693 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:28 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:28.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.824 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:28.824 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.824 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.824 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:28.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:28.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[23730]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:28.831 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[23730]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 815 B/s rd, 0 op/s 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[28528]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[28528]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 815 B/s rd, 0 op/s 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:28.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:28.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:28.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:28.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:28.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:28.835 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:28.548+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.835 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:28.835 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.836 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:28.837 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:28.838 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:28.839 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:28.840 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:28.840 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:28.840 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:28.840 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:28.840 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:28 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.726 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:29.726 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:29.726 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:29.726 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:29.727 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:29.728 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:29.729 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:29.730 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:29.731 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:29.732 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:29.732 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:29.732 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.732 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.732 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:29 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:29.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:29.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:29.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:29.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:29.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:29.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:29.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:29.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:29.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:29.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:29.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:29.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:29.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:29.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:29.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:29.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:29.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:29.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:29.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:29.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:29.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:29.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:29.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:29.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:29.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:29.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:29.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:29.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:29.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:29.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:29.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:29.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:29.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:29.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:29.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:29.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:29.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:29.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:29.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:29.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:29.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:29.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:29.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:29.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:29.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:29.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:29.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:29.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:29.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:29.835 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:29.836 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:29.836 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:29.836 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:29.836 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:29.836 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:29.837 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:29.837 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:29.837 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:29.837 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:29.837 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:29.837 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:29.838 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.838 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.838 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.838 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:29.839 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:29.839 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:29.839 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:29.839 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:29.839 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:29.839 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:29.840 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:29.840 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:29.840 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:29.840 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:29.840 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.840 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.841 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:29.841 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:29.841 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:29.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:29.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:29.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:29.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:29.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:29.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:29.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:29.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:29.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:29.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:29.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:29.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:29.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:29.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:29.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:29.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:29.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:29.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:29.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:29.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:29.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:29.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:29.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:29.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:29.846 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:29.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:29.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:29.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:29.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:29.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-09-13T22:16:29.847 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:29.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:29.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:29.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:29.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:29.848 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:29.849 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.849 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.849 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.849 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:29.850 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:29.850 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-09-13T22:16:29.850 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:29.850 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.850 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:29.850 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:29.851 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:29.851 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:29.851 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:29.851 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.851 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:29.851 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:29 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:30.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:30.276+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:30.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:30.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:30.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:30.532 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:30.533 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:30.534 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:30.535 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:30.536 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:30.537 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:30.537 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:30.289+0000 7f20e5479640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T22:16:30.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[23730]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:30.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[23730]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:30.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:30.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[28528]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[28528]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:16:30.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:30 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T22:16:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:30 smithi121 ceph-mon[81965]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:30 smithi121 ceph-mon[81965]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:30 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:30 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:30 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:16:30.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:30 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T22:16:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:31 smithi121 ceph-mon[81965]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:31.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:31.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:31.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:31.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:31.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:31.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:31.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:31 smithi121 ceph-mon[81965]: Upgrade: It appears safe to stop mon.b 2024-09-13T22:16:32.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:31 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:32.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:32.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:32.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:16:31] "GET /metrics HTTP/1.1" 200 37956 "" "Prometheus/2.43.0" 2024-09-13T22:16:32.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:31 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:32.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:32.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:32.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:32.078 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:32.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:32.080 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.081 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:31 smithi114 ceph-mon[23730]: Upgrade: It appears safe to stop mon.b 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:32.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:32.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:32.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:31 smithi114 ceph-mon[28528]: Upgrade: It appears safe to stop mon.b 2024-09-13T22:16:32.968 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:32 smithi121 ceph-mon[81965]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:33.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:32 smithi114 ceph-mon[23730]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:33.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:32 smithi114 ceph-mon[28528]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[23730]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[28528]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:16:35.107 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:34 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:34 smithi121 ceph-mon[81965]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:16:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:34 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:16:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:34 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:16:35.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:34 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:35.836 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:35 smithi121 ceph-mon[81965]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:35.836 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:35 smithi121 ceph-mon[81965]: Upgrade: Updating mon.b 2024-09-13T22:16:35.837 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:35 smithi121 ceph-mon[81965]: Deploying daemon mon.b on smithi121 2024-09-13T22:16:36.110 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:36.110 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[23730]: Upgrade: Updating mon.b 2024-09-13T22:16:36.110 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[23730]: Deploying daemon mon.b on smithi121 2024-09-13T22:16:36.110 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.110 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.110 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.110 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:36.111 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[28528]: Upgrade: Updating mon.b 2024-09-13T22:16:36.111 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[28528]: Deploying daemon mon.b on smithi121 2024-09-13T22:16:36.111 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.111 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.111 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:35 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:35 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:35 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:36.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:35 smithi121 ceph-mon[81965]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:37.000 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:36 smithi121 ceph-mon[81965]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:37.000 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:36 smithi121 ceph-mon[81965]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:37.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:36 smithi114 ceph-mon[23730]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:37.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:36 smithi114 ceph-mon[23730]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:37.246 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:36 smithi114 ceph-mon[28528]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:37.246 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:36 smithi114 ceph-mon[28528]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:38.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:37 smithi121 systemd[1]: Stopping Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:16:38.713 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b[81961]: 2024-09-13T22:16:38.401+0000 7feba8a53640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:16:38.713 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b[81961]: 2024-09-13T22:16:38.401+0000 7feba8a53640 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T22:16:38.988 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:38 smithi121 podman[83162]: 2024-09-13 22:16:38.712685419 +0000 UTC m=+0.403043231 container died 89a5df53ca799b21800e1cdf14138b1c4c905ffc524f71d79c9d285e77549099 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-13T22:16:39.312 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:39 smithi121 podman[83162]: 2024-09-13 22:16:39.086463347 +0000 UTC m=+0.776821157 container cleanup 89a5df53ca799b21800e1cdf14138b1c4c905ffc524f71d79c9d285e77549099 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be) 2024-09-13T22:16:39.312 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:39 smithi121 bash[83162]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b 2024-09-13T22:16:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:39 smithi121 podman[83192]: 2024-09-13 22:16:39.329563146 +0000 UTC m=+0.842799802 container remove 89a5df53ca799b21800e1cdf14138b1c4c905ffc524f71d79c9d285e77549099 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T22:16:40.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T22:16:40.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 systemd[1]: Stopped Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:16:40.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.b.service: Consumed 1.037s CPU time. 2024-09-13T22:16:40.349 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 systemd[1]: Starting Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:16:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 podman[83471]: 2024-09-13 22:16:40.336768812 +0000 UTC m=+0.020779165 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:16:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 podman[83471]: 2024-09-13 22:16:40.457936628 +0000 UTC m=+0.141946969 container create 2847ae29281d81fbd59b91a282062e41a951fe966b5a38337d7cc8a5b14757e1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 podman[83471]: 2024-09-13 22:16:40.66903893 +0000 UTC m=+0.353049271 container init 2847ae29281d81fbd59b91a282062e41a951fe966b5a38337d7cc8a5b14757e1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 podman[83471]: 2024-09-13 22:16:40.672383179 +0000 UTC m=+0.356393531 container start 2847ae29281d81fbd59b91a282062e41a951fe966b5a38337d7cc8a5b14757e1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: pidfile_write: ignore empty --pid-file 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: load: jerasure load: lrc 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: RocksDB version: 7.9.2 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Git sha 0 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T22:16:40.935 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: DB SUMMARY 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: DB Session ID: OAMEI6I5C6U8AE5L9DTM 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: MANIFEST file: MANIFEST-000074 size: 277 Bytes 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 2, files: 000069.sst 000072.sst 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000073.log size: 738812 ; 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T22:16:40.936 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.env: 0x55a9944ad940 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.info_log: 0x55a995f596e0 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.statistics: (nil) 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.use_fsync: 0 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:16:40.937 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.db_log_dir: 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.wal_dir: 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:16:40.938 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.write_buffer_manager: 0x55a995f688c0 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.unordered_write: 0 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:16:40.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.row_cache: None 2024-09-13T22:16:40.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.wal_filter: None 2024-09-13T22:16:40.940 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.wal_compression: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:16:40.941 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:16:40.942 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_open_files: -1 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:16:40.943 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Compression algorithms supported: 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kZSTD supported: 0 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000074 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:16:40.944 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.merge_operator: 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_filter: None 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a995f59320) 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_type: 0 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: data_block_index_type: 0 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_shortening: 1 2024-09-13T22:16:40.945 INFO:journalctl@ceph.mon.b.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: checksum: 4 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: no_block_cache: 0 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache: 0x55a995f4d1f0 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_options: 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: capacity : 536870912 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: num_shard_bits : 4 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: strict_capacity_limit : 0 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_cache_compressed: (nil) 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: persistent_cache: (nil) 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_size: 4096 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_size_deviation: 10 2024-09-13T22:16:40.946 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_restart_interval: 16 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: index_block_restart_interval: 1 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: metadata_block_size: 4096 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: partition_filters: 0 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: use_delta_encoding: 1 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: filter_policy: bloomfilter 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: whole_key_filtering: 1 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: verify_compression: 0 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: format_version: 5 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: enable_index_compression: 1 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: block_align: 0 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: max_auto_readahead_size: 262144 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: prepopulate_block_cache: 0 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: initial_auto_readahead_size: 8192 2024-09-13T22:16:40.947 INFO:journalctl@ceph.mon.b.smithi121.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression: NoCompression 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.num_levels: 7 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:16:40.948 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:16:40.949 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:16:40.950 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:16:40.951 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:16:40.952 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.ttl: 2592000 2024-09-13T22:16:40.953 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T22:16:40.954 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 69.sst 2024-09-13T22:16:40.955 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:16:40.955 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000074 succeeded,manifest_file_number is 74, next_file_number is 76, last_sequence is 37516, log_number is 68,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 68 2024-09-13T22:16:40.955 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 68 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 61c5307a-d4a1-4f7e-ad8a-c321313872c7 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265800699316, "job": 1, "event": "recovery_started", "wal_files": [73]} 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #73 mode 2 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265800701747, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 77, "file_size": 475068, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 37521, "largest_seqno": 37873, "table_properties": {"data_size": 472416, "index_size": 1164, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 4089, "raw_average_key_size": 24, "raw_value_size": 468683, "raw_average_value_size": 2823, "num_data_blocks": 49, "num_entries": 166, "num_filter_entries": 166, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726265800, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "61c5307a-d4a1-4f7e-ad8a-c321313872c7", "db_session_id": "OAMEI6I5C6U8AE5L9DTM", "orig_file_number": 77, "seqno_to_time_mapping": "N/A"}} 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265800701853, "job": 1, "event": "recovery_finished"} 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/version_set.cc:5047] Creating manifest 79 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000073.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a995f7ae00 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: DB pointer 0x55a996076000 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T22:16:40.956 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** DB Stats ** 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** Compaction Stats [default] ** 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: L0 2/0 756.48 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 341.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: L6 1/0 11.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T22:16:40.957 INFO:journalctl@ceph.mon.b.smithi121.stdout: Sum 3/0 11.75 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 341.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 341.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** Compaction Stats [default] ** 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 341.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:16:40.958 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: Cumulative compaction: 0.00 GB write, 95.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: Interval compaction: 0.00 GB write, 95.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: Block cache BinnedLRUCache@0x55a995f4d1f0#2 capacity: 512.00 MB usage: 3.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.94 KB,0.000178814%) IndexBlock(2,2.28 KB,0.000435114%) Misc(1,0.00 KB,0%) 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: starting mon.b rank 2 at public addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] at bind addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???) e3 preinit fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???).mds e1 new map 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???).mds e1 print_map 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: e1 2024-09-13T22:16:40.959 INFO:journalctl@ceph.mon.b.smithi121.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout: legacy client fscid: -1 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout: 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout: No filesystems configured 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: mon.b@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 bash[83471]: 2847ae29281d81fbd59b91a282062e41a951fe966b5a38337d7cc8a5b14757e1 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 systemd[1]: Started Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:16:40.960 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:40 smithi121 ceph-mon[83513]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 670 B/s rd, 0 op/s 2024-09-13T22:16:41.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:40 smithi114 ceph-mon[23730]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 670 B/s rd, 0 op/s 2024-09-13T22:16:41.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:40 smithi114 ceph-mon[28528]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 670 B/s rd, 0 op/s 2024-09-13T22:16:42.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:16:41] "GET /metrics HTTP/1.1" 200 37956 "" "Prometheus/2.43.0" 2024-09-13T22:16:42.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-09-13T22:16:42.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: mon.b calling monitor election 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: mon.c calling monitor election 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: mon.a calling monitor election 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: fsmap 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:42.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: mon.b calling monitor election 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: mon.c calling monitor election 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: mon.a calling monitor election 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: fsmap 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:16:42.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:42.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:42.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:42.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:42.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:41 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: mon.b calling monitor election 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: mon.c calling monitor election 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: mon.a calling monitor election 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: monmap e3: 3 mons at {a=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0],b=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],c=[v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0]} 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: fsmap 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T22:16:42.226 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:42.227 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:42.228 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:41 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:42.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:42.457+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:42.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:42.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:42.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:42.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:42.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:42.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:42.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:42.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:42.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:42.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:42.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:42.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.187 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:43.188 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:43.189 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:43.190 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:43.191 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:43.192 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:42 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:43.204 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:43.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:43.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:43.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:43.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:43.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:43.209 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:43.210 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:43.211 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:43.212 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:43.213 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:43.214 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:16:43.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:43.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:43.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:43.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:43.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:43.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:43.216 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:42 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:44.271 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:43 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:44.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:44.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:44.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:43.990+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:43 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.075 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:45.076 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:45.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:45.082 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.083 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:45.084 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:45.085 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:45.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:45.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:45.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:45.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:45.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:45.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:44 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:45.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:44 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:45.671+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:45.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:45.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:45.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:45.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:45.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:45.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:45.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:45.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:45.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:45.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:45.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:45.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:45.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:45.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:45.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:45.831 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:45.832 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:45 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:46.824 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:46.824 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:46.824 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:46.825 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:46.826 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:46.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:46.828 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:46.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:46.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:46.831 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:46.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:46.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:46.835 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:46.836 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:46.836 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:46.836 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:46.836 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.836 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.836 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:46 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.870 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:46 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:47.123+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:47.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:47.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:47.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:47.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:47.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:47.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:47.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:47.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:47.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:47.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:47.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:47.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:47.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:47.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:16:47.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:16:47.875 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:16:47.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:47.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:47.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:47.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:47.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:47 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:48.051 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.054 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.056 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.056 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.056 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T22:16:48.056 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.056 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.056 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.056 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:16:48.058 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:16:48.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:16:48.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.063 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:16:48.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:16:48.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:16:48.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:48.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:48.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:48.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:48.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:47 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:49.001 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:16:49.001 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:16:49.001 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.002 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:49.003 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.004 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.005 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all crash 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all mds 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all nfs 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:16:49.006 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all grafana 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all loki 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all promtail 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Finalizing container_image settings 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Upgrade: Complete! 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:49.007 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:48 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:49.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:16:49.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:16:49.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.075 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:49.076 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:49.077 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.078 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.079 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.080 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.081 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.082 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:49.083 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.084 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.085 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.086 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.086 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all crash 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all mds 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all nfs 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all grafana 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all loki 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all promtail 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Finalizing container_image settings 2024-09-13T22:16:49.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Upgrade: Complete! 2024-09-13T22:16:49.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:48.796+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.089 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.090 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all crash 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all mds 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:16:49.091 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all nfs 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all grafana 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all loki 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all promtail 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Finalizing container_image settings 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Upgrade: Complete! 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.092 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:49.093 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.094 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.095 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.096 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.096 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.096 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.096 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.853 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.853 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.853 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.853 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.853 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.854 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:49.855 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.856 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.857 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:49.858 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.859 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.860 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:49.861 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:49.862 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:49.863 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:49.864 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:49.865 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:49.865 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.865 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:49.865 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:49.865 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:49.865 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:49 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:50.100 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:50.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:50.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:50.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:50.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:50.100 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:50.101 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:50.102 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:50.103 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:50.104 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.105 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:50.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:16:50.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:50.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:16:50.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:50.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:16:50.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:49 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:16:50.272+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:50.561 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:50.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:50.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:50.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.564 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:50.564 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:50.564 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:50.564 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:50.565 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:16:50.565 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:50.565 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:50.565 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:50.565 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:16:50.565 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:50.566 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:16:50.566 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:50.566 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:16:50.566 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:50.566 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:16:50.567 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:50.567 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:16:50.567 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:50.567 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:16:50.567 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:50.567 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:50.568 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:50.568 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.568 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.568 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:16:50.568 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:50.569 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:16:50.569 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:50.569 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:16:50.569 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:50.569 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:50.570 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:50.570 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:50.570 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:50.570 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:50.570 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:50.570 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:50.571 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:50.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:50.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:50.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:50.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:50 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:50 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:16:51.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:50 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:16:51.393 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:16:52.044 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:16:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:16:51] "GET /metrics HTTP/1.1" 200 37953 "" "Prometheus/2.43.0" 2024-09-13T22:16:52.044 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:51 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:52.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:52.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:51 smithi114 ceph-mon[23730]: from='client.25597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:52.048 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:51 smithi114 ceph-mon[23730]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:51 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:52.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:52.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:52.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:52.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:52.053 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.053 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:51 smithi114 ceph-mon[28528]: from='client.25597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:52.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:51 smithi114 ceph-mon[28528]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:51 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:16:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:16:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:16:52.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:16:52.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:51 smithi121 ceph-mon[83513]: from='client.25597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:52.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:51 smithi121 ceph-mon[83513]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:53.211 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:52 smithi114 ceph-mon[23730]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-09-13T22:16:53.211 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:52 smithi114 ceph-mon[28528]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-09-13T22:16:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:52 smithi121 ceph-mon[83513]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 18s ago 29m - - 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 6s ago 29m - - 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (15m) 18s ago 24m 25.9M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (80s) 6s ago 23m 76.3M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 18s ago 22m 51.2M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (2m) 6s ago 28m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (12m) 18s ago 30m 518M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (30m) 18s ago 30m 87.8M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (13s) 6s ago 28m 21.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:16:53.541 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (29m) 18s ago 29m 62.5M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (16m) 18s ago 24m 14.6M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (15m) 6s ago 24m 15.3M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (28m) 18s ago 28m 57.5M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (28m) 18s ago 28m 57.6M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (27m) 18s ago 27m 50.7M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (27m) 18s ago 27m 51.9M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (26m) 6s ago 26m 54.6M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (26m) 6s ago 26m 54.8M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (26m) 6s ago 26m 52.7M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (25m) 6s ago 25m 53.1M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (2m) 6s ago 25m 67.0M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (23m) 18s ago 23m 76.4M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:16:53.542 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (23m) 6s ago 23m 77.5M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:16:54.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:53 smithi114 ceph-mon[23730]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:54.259 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:53 smithi114 ceph-mon[28528]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:54.284 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-09-13T22:16:54.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:53 smithi121 ceph-mon[83513]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:16:55.038 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:54 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:55.039 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:54 smithi114 ceph-mon[23730]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:55.039 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:54 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:55.039 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:54 smithi114 ceph-mon[28528]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:54 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:16:55.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:54 smithi121 ceph-mon[83513]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:56.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[23730]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:56.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[23730]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:56.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.207 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[28528]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:56.207 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[28528]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:56.207 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:56 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:56 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:56 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:56 smithi121 ceph-mon[83513]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:16:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:56 smithi121 ceph-mon[83513]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:16:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:56 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:16:57.744 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:16:58.272 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T22:16:58.295 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:58 smithi114 ceph-mon[23730]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:58.295 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:58 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/2096166172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:58.295 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:58 smithi114 ceph-mon[28528]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:58.296 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:58 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2096166172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:58 smithi121 ceph-mon[83513]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:16:58.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:58 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/2096166172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:16:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:16:59 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:59.148 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:16:59 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:16:59.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:16:59 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:00.057 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:17:00.057 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": null, 2024-09-13T22:17:00.057 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": false, 2024-09-13T22:17:00.058 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "", 2024-09-13T22:17:00.058 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:17:00.058 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": null, 2024-09-13T22:17:00.058 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "", 2024-09-13T22:17:00.058 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:17:00.058 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:17:00.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:00 smithi114 ceph-mon[23730]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 622 B/s rd, 0 op/s 2024-09-13T22:17:00.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:00 smithi114 ceph-mon[28528]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 622 B/s rd, 0 op/s 2024-09-13T22:17:00.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:00 smithi121 ceph-mon[83513]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 622 B/s rd, 0 op/s 2024-09-13T22:17:00.696 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T22:17:02.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:01 smithi114 ceph-mon[28528]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:02.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:01 smithi114 ceph-mon[23730]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:02.064 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:17:01] "GET /metrics HTTP/1.1" 200 38021 "" "Prometheus/2.43.0" 2024-09-13T22:17:02.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:01 smithi121 ceph-mon[83513]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:02.846 INFO:teuthology.orchestra.run.smithi114.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:17:02.846 INFO:teuthology.orchestra.run.smithi114.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:17:02.847 INFO:teuthology.orchestra.run.smithi114.stdout: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:17:03.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:02 smithi121 ceph-mon[83513]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:03.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:02 smithi114 ceph-mon[23730]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:03.241 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:02 smithi114 ceph-mon[28528]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:03.468 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-09-13T22:17:04.019 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:03 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/3234092319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:17:04.020 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:03 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/3234092319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:17:04.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:03 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/3234092319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:17:04.856 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:04 smithi114 ceph-mon[23730]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 622 B/s rd, 0 op/s 2024-09-13T22:17:04.857 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:04 smithi114 ceph-mon[28528]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 622 B/s rd, 0 op/s 2024-09-13T22:17:05.109 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:04 smithi121 ceph-mon[83513]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 622 B/s rd, 0 op/s 2024-09-13T22:17:06.036 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:05 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:06.037 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:05 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:06.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:05 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:07.088 INFO:teuthology.orchestra.run.smithi114.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:07.104 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:06 smithi114 ceph-mon[28528]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:17:07.104 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:06 smithi114 ceph-mon[28528]: from='client.25627 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:07.104 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:06 smithi114 ceph-mon[28528]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi114", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:07.104 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:06 smithi114 ceph-mon[23730]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:17:07.104 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:06 smithi114 ceph-mon[23730]: from='client.25627 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:07.104 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:06 smithi114 ceph-mon[23730]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi114", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:06 smithi121 ceph-mon[83513]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:17:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:06 smithi121 ceph-mon[83513]: from='client.25627 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:07.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:06 smithi121 ceph-mon[83513]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "mon", "hosts": "smithi114", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:07.696 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-09-13T22:17:08.311 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.312 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:08.312 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:08.312 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:08.312 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:08 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:08.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:08.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[23730]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:08.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:08.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-09-13T22:17:08.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:08.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:08.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:08.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:08.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:08.944+0000 7f20e2c74640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:09.149 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:09.150 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:09.151 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:09.152 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:09.153 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:10 smithi114 ceph-mon[23730]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.202 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.203 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:10.204 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:10.205 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:10 smithi114 ceph-mon[23730]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:10.206 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:10 smithi114 ceph-mon[23730]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:10 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:10.207 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:10.208 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:10.209 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:10.210 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:10.211 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:10.211 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:10.211 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:10.211 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.211 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.211 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:10 smithi114 ceph-mon[28528]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:10.211 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:10 smithi114 ceph-mon[28528]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:17:10.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:10 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:10.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:10.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:10.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:10.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:10.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:10.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:10.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:10 smithi121 ceph-mon[83513]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:10.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:10 smithi121 ceph-mon[83513]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:17:10.658 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:17:11.324 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:17:11.324 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 16s ago 29m - - 2024-09-13T22:17:11.324 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 4s ago 29m - - 2024-09-13T22:17:11.324 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (16m) 16s ago 24m 25.9M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:17:11.324 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (98s) 4s ago 24m 76.4M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 16s ago 23m 52.8M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (2m) 4s ago 29m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (12m) 16s ago 30m 518M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (30m) 16s ago 30m 89.4M 2048M 17.2.0 e1d6a67b021e dae4b66af8ad 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (30s) 4s ago 29m 29.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (29m) 16s ago 29m 64.8M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (16m) 16s ago 25m 14.8M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (15m) 4s ago 25m 14.6M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (28m) 16s ago 28m 57.5M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (28m) 16s ago 28m 57.6M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (27m) 16s ago 27m 50.7M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (27m) 16s ago 27m 51.9M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:17:11.325 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (27m) 4s ago 27m 54.6M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:17:11.326 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (26m) 4s ago 26m 54.8M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:17:11.326 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (26m) 4s ago 26m 52.7M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:17:11.326 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (26m) 4s ago 26m 53.1M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:17:11.326 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (3m) 4s ago 25m 67.1M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:17:11.326 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (23m) 16s ago 23m 76.4M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:17:11.326 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (23m) 4s ago 23m 77.6M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:17:11.757 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:17:11] "GET /metrics HTTP/1.1" 200 38021 "" "Prometheus/2.43.0" 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:17:11.778 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:17:11.779 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:17:11.779 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:11.779 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:17:11.779 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T22:17:11.779 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T22:17:11.779 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:17:11.779 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:17:12.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:12.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:12.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:12.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:12.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:12.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:17:12.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[23730]: from='client.? 172.21.15.114:0/4135592699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T22:17:12.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:11 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/4135592699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:12.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:17:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T22:17:12.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:11 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/4135592699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:12.161 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi114", 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "0/2 daemons upgraded", 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:17:12.162 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:17:12.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[23730]: from='client.15759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:12.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[23730]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:17:12.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[23730]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:12.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[23730]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:17:12.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[23730]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:17:12.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[23730]: Upgrade: Setting container_image for all mgr 2024-09-13T22:17:12.937 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[23730]: Upgrade: It appears safe to stop mon.a 2024-09-13T22:17:12.938 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[28528]: from='client.15759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:12.938 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[28528]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:17:12.938 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[28528]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:12.938 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[28528]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:17:12.938 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[28528]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:17:12.938 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[28528]: Upgrade: Setting container_image for all mgr 2024-09-13T22:17:12.938 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:12 smithi114 ceph-mon[28528]: Upgrade: It appears safe to stop mon.a 2024-09-13T22:17:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:12 smithi121 ceph-mon[83513]: from='client.15759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:12 smithi121 ceph-mon[83513]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-09-13T22:17:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:12 smithi121 ceph-mon[83513]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:12 smithi121 ceph-mon[83513]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:17:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:12 smithi121 ceph-mon[83513]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:17:13.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:12 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all mgr 2024-09-13T22:17:13.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:12 smithi121 ceph-mon[83513]: Upgrade: It appears safe to stop mon.a 2024-09-13T22:17:13.943 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:13 smithi114 ceph-mon[23730]: from='client.15777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:13.943 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:13 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:13.944 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:13 smithi114 ceph-mon[28528]: from='client.15777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:13.944 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:13 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:13 smithi121 ceph-mon[83513]: from='client.15777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:14.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:13 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:14 smithi121 ceph-mon[83513]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-09-13T22:17:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:14 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:14 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:17:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:14 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:17:15.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:14 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:15.196 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[23730]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-09-13T22:17:15.196 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:15.196 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:17:15.196 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:17:15.196 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:15.196 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[28528]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-09-13T22:17:15.197 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:15.197 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:17:15.197 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:17:15.197 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:14 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:15 smithi121 ceph-mon[83513]: Upgrade: Updating mon.a 2024-09-13T22:17:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:15 smithi121 ceph-mon[83513]: Deploying daemon mon.a on smithi114 2024-09-13T22:17:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:15 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:15 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:15 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:15 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.316 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[23730]: Upgrade: Updating mon.a 2024-09-13T22:17:16.316 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[23730]: Deploying daemon mon.a on smithi114 2024-09-13T22:17:16.316 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[23730]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:16.316 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.316 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.317 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[23730]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.317 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[28528]: Upgrade: Updating mon.a 2024-09-13T22:17:16.317 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[28528]: Deploying daemon mon.a on smithi114 2024-09-13T22:17:16.317 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:16.317 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.317 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:16.317 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:15 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:16 smithi114 ceph-mon[28528]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:16 smithi114 ceph-mon[28528]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:16 smithi114 ceph-mon[28528]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:16 smithi114 ceph-mon[23730]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:16 smithi114 ceph-mon[23730]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:16 smithi114 ceph-mon[23730]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:16 smithi114 systemd[1]: Stopping Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[23726]: 2024-09-13T22:17:17.101+0000 7f593a236700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:17:17.109 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[23726]: 2024-09-13T22:17:17.101+0000 7f593a236700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T22:17:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:16 smithi121 ceph-mon[83513]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:16 smithi121 ceph-mon[83513]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:16 smithi121 ceph-mon[83513]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:17.562 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:17 smithi114 podman[102319]: 2024-09-13 22:17:17.286131294 +0000 UTC m=+0.275167078 container died dae4b66af8adf6d52f1b5df3b3d9813dbda4db14e34e73fb246214ffa6903df3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T22:17:17.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:17] ENGINE Bus STOPPING 2024-09-13T22:17:17.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:17:17.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:17] ENGINE Bus STOPPED 2024-09-13T22:17:17.562 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:17] ENGINE Bus STARTING 2024-09-13T22:17:17.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:17] ENGINE Serving on http://:::9283 2024-09-13T22:17:17.563 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:17] ENGINE Bus STARTED 2024-09-13T22:17:17.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:17 smithi114 podman[102319]: 2024-09-13 22:17:17.660925163 +0000 UTC m=+0.649960977 container cleanup dae4b66af8adf6d52f1b5df3b3d9813dbda4db14e34e73fb246214ffa6903df3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.expose-services=, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.buildah.version=1.19.8) 2024-09-13T22:17:17.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:17 smithi114 bash[102319]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a 2024-09-13T22:17:18.194 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:17 smithi114 podman[102342]: 2024-09-13 22:17:17.928997015 +0000 UTC m=+0.642273719 container remove dae4b66af8adf6d52f1b5df3b3d9813dbda4db14e34e73fb246214ffa6903df3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_CLEAN=True, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.buildah.version=1.19.8) 2024-09-13T22:17:18.194 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T22:17:18.194 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 systemd[1]: Stopped Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:17:18.194 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service: Consumed 28.006s CPU time. 2024-09-13T22:17:18.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 systemd[1]: Starting Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:17:18.766 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 podman[102472]: 2024-09-13 22:17:18.405047332 +0000 UTC m=+0.016298416 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:18.766 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 podman[102472]: 2024-09-13 22:17:18.505101058 +0000 UTC m=+0.116352204 container create c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T22:17:18.766 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 podman[102472]: 2024-09-13 22:17:18.740962939 +0000 UTC m=+0.352214024 container init c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T22:17:18.766 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 podman[102472]: 2024-09-13 22:17:18.744401184 +0000 UTC m=+0.355652269 container start c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T22:17:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: pidfile_write: ignore empty --pid-file 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: load: jerasure load: lrc 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: RocksDB version: 7.9.2 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Git sha 0 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: DB SUMMARY 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: DB Session ID: YKNR0KCKLHFEOAWSWKSW 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:17:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3866 Bytes 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000075.sst 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000073.log size: 2106981 ; 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.env: 0x55b4e8f0c940 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T22:17:19.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.info_log: 0x55b4eb4536e0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.statistics: (nil) 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.use_fsync: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:17:19.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.db_log_dir: 2024-09-13T22:17:19.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.wal_dir: 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.write_buffer_manager: 0x55b4eb4628c0 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:17:19.079 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.unordered_write: 0 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.row_cache: None 2024-09-13T22:17:19.080 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.wal_filter: None 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.wal_compression: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:17:19.081 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:17:19.082 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_open_files: -1 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Compression algorithms supported: 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kZSTD supported: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:17:19.083 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.merge_operator: 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_filter: None 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b4eb453320) 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:17:19.084 INFO:journalctl@ceph.mon.a.smithi114.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_type: 0 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: data_block_index_type: 0 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_shortening: 1 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: checksum: 4 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: no_block_cache: 0 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache: 0x55b4eb4471f0 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_options: 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: capacity : 536870912 2024-09-13T22:17:19.085 INFO:journalctl@ceph.mon.a.smithi114.stdout: num_shard_bits : 4 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: strict_capacity_limit : 0 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_compressed: (nil) 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: persistent_cache: (nil) 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_size: 4096 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_size_deviation: 10 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_restart_interval: 16 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_block_restart_interval: 1 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: metadata_block_size: 4096 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: partition_filters: 0 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: use_delta_encoding: 1 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: filter_policy: bloomfilter 2024-09-13T22:17:19.086 INFO:journalctl@ceph.mon.a.smithi114.stdout: whole_key_filtering: 1 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: verify_compression: 0 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: format_version: 5 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: enable_index_compression: 1 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_align: 0 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: max_auto_readahead_size: 262144 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: prepopulate_block_cache: 0 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: initial_auto_readahead_size: 8192 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression: NoCompression 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:17:19.087 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.num_levels: 7 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:17:19.088 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:17:19.089 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:17:19.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:17:19.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:17:19.090 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:17:19.092 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:17:19.092 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:17:19.092 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:17:19.092 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:17:19.092 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:17:19.092 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:17:19.093 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:17:19.094 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.ttl: 2592000 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:17:19.095 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 75.sst 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 77, last_sequence is 34477, log_number is 73,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 73 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3b5bcef9-ab59-4baf-9176-b2df0666f9a5 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265838771425, "job": 1, "event": "recovery_started", "wal_files": [73]} 2024-09-13T22:17:19.096 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #73 mode 2 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265838778083, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 78, "file_size": 1915988, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 34471, "largest_seqno": 36148, "table_properties": {"data_size": 1908256, "index_size": 4642, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2053, "raw_key_size": 20047, "raw_average_key_size": 25, "raw_value_size": 1891643, "raw_average_value_size": 2379, "num_data_blocks": 205, "num_entries": 795, "num_filter_entries": 795, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726265838, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3b5bcef9-ab59-4baf-9176-b2df0666f9a5", "db_session_id": "YKNR0KCKLHFEOAWSWKSW", "orig_file_number": 78, "seqno_to_time_mapping": "N/A"}} 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265838778187, "job": 1, "event": "recovery_finished"} 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/version_set.cc:5047] Creating manifest 80 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000073.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b4eb474e00 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: DB pointer 0x55b4eb570000 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T22:17:19.097 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** DB Stats ** 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:17:19.098 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: L0 1/0 1.83 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 541.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: L6 1/0 11.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: Sum 2/0 12.84 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 541.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 541.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 541.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T22:17:19.099 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative compaction: 0.00 GB write, 199.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval compaction: 0.00 GB write, 199.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Block cache BinnedLRUCache@0x55b4eb4471f0#2 capacity: 512.00 MB usage: 7.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T22:17:19.100 INFO:journalctl@ceph.mon.a.smithi114.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: starting mon.a rank 0 at public addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] at bind addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???) e3 preinit fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???).mds e1 new map 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???).mds e1 print_map 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: e1 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: legacy client fscid: -1 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout: No filesystems configured 2024-09-13T22:17:19.101 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T22:17:19.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:17:19.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:17:19.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:17:19.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 ceph-mon[102491]: mon.a@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-13T22:17:19.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 bash[102472]: c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 2024-09-13T22:17:19.102 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:18 smithi114 systemd[1]: Started Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: ignoring --setuser ceph since I am not root 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: ignoring --setgroup ceph since I am not root 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mgr[75522]: -- 172.21.15.121:0/130992871 <== mon.2 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b97668d0e0 con 0x55b97666b000 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:20.359+0000 7f5536f21040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:20.416+0000 7f5536f21040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:17:20.565 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: mon.a calling monitor election 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: monmap epoch 3 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: last_changed 2024-09-13T21:47:54.401708+0000 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: min_mon_release 17 (quincy) 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: election_strategy: 1 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: fsmap 2024-09-13T22:17:20.566 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:17:20.567 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: mgrmap e42: y(active, since 3m), standbys: x 2024-09-13T22:17:20.567 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:17:20.567 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:17:20.567 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:17:20.567 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: from='mgr.25219 ' entity='' 2024-09-13T22:17:20.567 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-mon[83513]: mgrmap e43: y(active, since 3m), standbys: x 2024-09-13T22:17:20.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ignoring --setuser ceph since I am not root 2024-09-13T22:17:20.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ignoring --setgroup ceph since I am not root 2024-09-13T22:17:20.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mgr[70608]: -- 172.21.15.114:0/4221257180 <== mon.0 v2:172.21.15.114:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5623b3f610e0 con 0x5623b3f3f000 2024-09-13T22:17:20.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:20.372+0000 7fce6ccc2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T22:17:20.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:20.429+0000 7fce6ccc2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T22:17:20.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: mon.a calling monitor election 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: monmap epoch 3 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: last_changed 2024-09-13T21:47:54.401708+0000 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: min_mon_release 17 (quincy) 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: election_strategy: 1 2024-09-13T22:17:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: fsmap 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: mgrmap e42: y(active, since 3m), standbys: x 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: from='mgr.25219 ' entity='' 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[102491]: mgrmap e43: y(active, since 3m), standbys: x 2024-09-13T22:17:20.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: from='mgr.25219 172.21.15.114:0/800367472' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: mon.a calling monitor election 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: monmap epoch 3 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: last_changed 2024-09-13T21:47:54.401708+0000 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:17:20.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: min_mon_release 17 (quincy) 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: election_strategy: 1 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: fsmap 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: mgrmap e42: y(active, since 3m), standbys: x 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: from='mgr.25219 ' entity='' 2024-09-13T22:17:20.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-mon[28528]: mgrmap e43: y(active, since 3m), standbys: x 2024-09-13T22:17:20.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:20.563+0000 7f5536f21040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:17:20.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:20.629+0000 7f5536f21040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:17:20.871 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:20.778+0000 7f5536f21040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:17:20.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:20.576+0000 7fce6ccc2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T22:17:20.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:20.642+0000 7fce6ccc2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T22:17:20.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:20.789+0000 7fce6ccc2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:17:20.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:20.929+0000 7fce6ccc2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:17:21.232 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:20.918+0000 7f5536f21040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T22:17:21.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:21.244+0000 7fce6ccc2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:17:21.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:21.305+0000 7fce6ccc2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:17:21.574 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:21.490+0000 7fce6ccc2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:17:21.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:21.231+0000 7f5536f21040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T22:17:21.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:21.292+0000 7f5536f21040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T22:17:21.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:21.474+0000 7f5536f21040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T22:17:22.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:22 smithi114 ceph-mon[102491]: mgrmap e44: y(active, since 3m), standbys: x 2024-09-13T22:17:22.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:22 smithi114 ceph-mon[28528]: mgrmap e44: y(active, since 3m), standbys: x 2024-09-13T22:17:22.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:22.380+0000 7fce6ccc2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:17:22.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:22 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:22.448+0000 7fce6ccc2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:17:22.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:22 smithi121 ceph-mon[83513]: mgrmap e44: y(active, since 3m), standbys: x 2024-09-13T22:17:22.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:22.368+0000 7f5536f21040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T22:17:22.621 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:22.435+0000 7f5536f21040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T22:17:23.397 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.140+0000 7f5536f21040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:17:23.397 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.254+0000 7f5536f21040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:17:23.420 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.160+0000 7fce6ccc2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T22:17:23.421 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.275+0000 7fce6ccc2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T22:17:23.672 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.395+0000 7f5536f21040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:17:23.673 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.456+0000 7f5536f21040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:17:23.673 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.603+0000 7f5536f21040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:17:23.673 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.671+0000 7f5536f21040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:17:23.697 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.419+0000 7fce6ccc2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T22:17:23.697 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.480+0000 7fce6ccc2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T22:17:23.697 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.628+0000 7fce6ccc2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T22:17:23.955 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.731+0000 7f5536f21040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:17:23.955 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.890+0000 7f5536f21040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:17:23.997 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.695+0000 7fce6ccc2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T22:17:23.997 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.754+0000 7fce6ccc2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T22:17:23.997 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.929+0000 7fce6ccc2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T22:17:23.997 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:23 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:23.994+0000 7fce6ccc2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:17:24.221 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:23.954+0000 7f5536f21040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T22:17:24.221 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:24.020+0000 7f5536f21040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:17:24.221 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:24.102+0000 7f5536f21040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:17:24.221 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:24.161+0000 7f5536f21040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:17:24.221 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:24.220+0000 7f5536f21040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:17:24.263 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:24.060+0000 7fce6ccc2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T22:17:24.263 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:24.142+0000 7fce6ccc2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T22:17:24.263 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:24.202+0000 7fce6ccc2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T22:17:24.263 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:24.261+0000 7fce6ccc2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T22:17:24.263 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:17:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:17:24.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:17:24.263 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:17:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:17:24.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:17:25.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[28528]: Standby manager daemon x restarted 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[28528]: Standby manager daemon x started 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[28528]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[102491]: Standby manager daemon x restarted 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[102491]: Standby manager daemon x started 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[102491]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:17:25.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[102491]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:17:25.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[102491]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:17:25.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-mon[102491]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:17:25.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:25.100+0000 7fce6ccc2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:17:25.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:25] ENGINE Bus STARTING 2024-09-13T22:17:25.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: CherryPy Checker: 2024-09-13T22:17:25.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: The Application mounted at '' has an empty config. 2024-09-13T22:17:25.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-mon[83513]: Standby manager daemon x restarted 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-mon[83513]: Standby manager daemon x started 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-mon[83513]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-mon[83513]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-mon[83513]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-mon[83513]: from='mgr.? 172.21.15.121:0/3595613985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:25.005+0000 7f5536f21040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: [13/Sep/2024:22:17:25] ENGINE Bus STARTING 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: CherryPy Checker: 2024-09-13T22:17:25.327 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: The Application mounted at '' has an empty config. 2024-09-13T22:17:25.328 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: 2024-09-13T22:17:25.328 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: [13/Sep/2024:22:17:25] ENGINE Serving on http://:::9283 2024-09-13T22:17:25.328 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:17:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x[75518]: [13/Sep/2024:22:17:25] ENGINE Bus STARTED 2024-09-13T22:17:25.602 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:25] ENGINE Serving on http://:::9283 2024-09-13T22:17:25.602 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:17:25] ENGINE Bus STARTED 2024-09-13T22:17:26.356 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:26.121+0000 7fce0b9cb640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: mgrmap e45: y(active, since 3m), standbys: x 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: Active manager daemon y restarted 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: Activating manager daemon y 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: mgrmap e46: y(active, starting, since 0.00699683s), standbys: x 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:17:26.357 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: Manager daemon y is now available 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.358 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:26.359 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: mgrmap e45: y(active, since 3m), standbys: x 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: Active manager daemon y restarted 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: Activating manager daemon y 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: mgrmap e46: y(active, starting, since 0.00699683s), standbys: x 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:17:26.360 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: Manager daemon y is now available 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:26.361 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:26.362 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: mgrmap e45: y(active, since 3m), standbys: x 2024-09-13T22:17:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: Active manager daemon y restarted 2024-09-13T22:17:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: Activating manager daemon y 2024-09-13T22:17:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:17:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: mgrmap e46: y(active, starting, since 0.00699683s), standbys: x 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:17:26.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: Manager daemon y is now available 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T22:17:26.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.374 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:26.374 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.374 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:17:26.374 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:26.374 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:26.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:17:26.355Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2024-09-13T22:17:26.824 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:17:26.461Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-09-13T22:17:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:17:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:17:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: mgrmap e47: y(active, since 1.01103s), standbys: x 2024-09-13T22:17:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:27.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: mgrmap e47: y(active, since 1.01103s), standbys: x 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:17:27.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:27 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:26.890+0000 7fcdc4275640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:22:17:26] ENGINE Error in HTTPServer.serve 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._connections.run(self.expiration_interval) 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._run(expiration_interval) 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:17:27.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: s = self.context.wrap_socket( 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.sslsocket_class._create( 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.do_handshake() 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self._sslobj.do_handshake() 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:17:27.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: Updating smithi114:/etc/ceph/ceph.conf 2024-09-13T22:17:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: Updating smithi121:/etc/ceph/ceph.conf 2024-09-13T22:17:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: mgrmap e47: y(active, since 1.01103s), standbys: x 2024-09-13T22:17:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:17:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:27.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:27 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.289 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:17:28.289 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:17:28.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: [13/Sep/2024:22:17:26] ENGINE Bus STARTING 2024-09-13T22:17:28.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:17:28.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:17:28.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: [13/Sep/2024:22:17:26] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:17:28.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: [13/Sep/2024:22:17:26] ENGINE Serving on https://172.21.15.114:7151 2024-09-13T22:17:28.290 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: [13/Sep/2024:22:17:26] ENGINE Bus STARTED 2024-09-13T22:17:28.291 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: [13/Sep/2024:22:17:26] ENGINE Error in HTTPServer.serve 2024-09-13T22:17:28.291 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:28.291 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:17:28.291 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:17:28.291 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:17:28.291 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._run(expiration_interval) 2024-09-13T22:17:28.292 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:17:28.292 INFO:journalctl@ceph.mon.b.smithi121.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:17:28.292 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:17:28.292 INFO:journalctl@ceph.mon.b.smithi121.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:17:28.292 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:17:28.292 INFO:journalctl@ceph.mon.b.smithi121.stdout: s = self.context.wrap_socket( 2024-09-13T22:17:28.293 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:17:28.293 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.sslsocket_class._create( 2024-09-13T22:17:28.293 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:17:28.293 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.do_handshake() 2024-09-13T22:17:28.293 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:17:28.293 INFO:journalctl@ceph.mon.b.smithi121.stdout: self._sslobj.do_handshake() 2024-09-13T22:17:28.294 INFO:journalctl@ceph.mon.b.smithi121.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:17:28.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:17:28.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T22:17:28.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: Cluster is now healthy 2024-09-13T22:17:28.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:28.294 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.295 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.295 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:28.295 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.295 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.295 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:17:28.295 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:17:28.296 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:28.296 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:28.296 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:28.296 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:28 smithi121 ceph-mon[83513]: mgrmap e48: y(active, since 2s), standbys: x 2024-09-13T22:17:28.322 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: [13/Sep/2024:22:17:26] ENGINE Bus STARTING 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: [13/Sep/2024:22:17:26] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: [13/Sep/2024:22:17:26] ENGINE Serving on https://172.21.15.114:7151 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: [13/Sep/2024:22:17:26] ENGINE Bus STARTED 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: [13/Sep/2024:22:17:26] ENGINE Error in HTTPServer.serve 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:17:28.323 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.do_handshake() 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:17:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: Cluster is now healthy 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:28.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[28528]: mgrmap e48: y(active, since 2s), standbys: x 2024-09-13T22:17:28.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:17:28.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:17:28.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: [13/Sep/2024:22:17:26] ENGINE Bus STARTING 2024-09-13T22:17:28.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:17:28.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: Updating smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:17:28.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: [13/Sep/2024:22:17:26] ENGINE Serving on http://172.21.15.114:8765 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: [13/Sep/2024:22:17:26] ENGINE Serving on https://172.21.15.114:7151 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: [13/Sep/2024:22:17:26] ENGINE Bus STARTED 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: [13/Sep/2024:22:17:26] ENGINE Error in HTTPServer.serve 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._run(expiration_interval) 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:17:28.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: s = self.context.wrap_socket( 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.sslsocket_class._create( 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.do_handshake() 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: self._sslobj.do_handshake() 2024-09-13T22:17:28.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: Updating smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: Cluster is now healthy 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:17:28.329 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi114", "caps": []}]: dispatch 2024-09-13T22:17:28.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:28.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:28.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:28.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:28 smithi114 ceph-mon[102491]: mgrmap e48: y(active, since 2s), standbys: x 2024-09-13T22:17:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:29 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:29 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:29 smithi121 ceph-mon[83513]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:17:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:29 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:29.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:29 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:29.390 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[102491]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:29.390 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[102491]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:29.390 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[102491]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:17:29.390 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:29.390 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:29.390 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:29.390 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:29.391 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[28528]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T22:17:29.391 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:29.391 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:29 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:30.273 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-mon[102491]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:30.273 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-mon[102491]: mgrmap e49: y(active, since 4s), standbys: x 2024-09-13T22:17:30.273 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-mon[28528]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:30.273 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-mon[28528]: mgrmap e49: y(active, since 4s), standbys: x 2024-09-13T22:17:30.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:30.319+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:30.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:30.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:30.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:30.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:30.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:30.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:30.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:30.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:30.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:30.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:17:30.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:30.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:30.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:30.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:30 smithi121 ceph-mon[83513]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:30.338 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:30 smithi121 ceph-mon[83513]: mgrmap e49: y(active, since 4s), standbys: x 2024-09-13T22:17:31.282 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.283 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:31 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:31 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi121", "caps": []}]: dispatch 2024-09-13T22:17:31.985 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:17:31] "GET /metrics HTTP/1.1" 200 35221 "" "Prometheus/2.43.0" 2024-09-13T22:17:32.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:32.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:32.324 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:32.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:32.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:32.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.331 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-13T22:17:32.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:32.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:32.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:17:32.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:32.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:32.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:32.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:32.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:32.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:32.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:32.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:32.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:32.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:32.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:32.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:32.335 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:32.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:32.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:32.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:32.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:32.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:32.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:32.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:32.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-13T22:17:32.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:32.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:32.337 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:32.338 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:32.338 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:32.338 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:32.338 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:32.338 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:32.338 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:32.339 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:32.339 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:32.339 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:32 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:32.484 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:32.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:32.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:32.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:32.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:32.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.485 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:32.486 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:32.487 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:32.487 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:32.487 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:32.487 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:32.487 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.487 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.488 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.488 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:32.488 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:32.488 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:32.488 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:17:32.488 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:32.489 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:32.489 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:32.489 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:17:32.489 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:32.489 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:17:32.490 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:32.490 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:17:32.490 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:32.490 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:32.490 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:32.491 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:17:32.491 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:32.491 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:32.491 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:32.491 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:32.491 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:32.491 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.492 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.492 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:32.492 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:32.492 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:32.492 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:32.492 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:32.493 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:32.493 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:32.494 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:32.494 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:32.494 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:32.494 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:32.494 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:32.494 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:32.495 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:32.495 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-13T22:17:32.495 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:32.495 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:32.495 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:32.495 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:32.496 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:32.496 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.413 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.414 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:33 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:33.471 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:33 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.344 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:34 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.361 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:34 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.361 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:34 smithi121 ceph-mon[83513]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T22:17:34.361 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:34 smithi121 ceph-mon[83513]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T22:17:34.362 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:34 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.362 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:34 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.565 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.565 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.565 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[102491]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T22:17:34.565 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[102491]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T22:17:34.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[28528]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T22:17:34.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[28528]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T22:17:34.567 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:34.567 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:34 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-13T22:17:35.445 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:35.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.446 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:35 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-13T22:17:35.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:35.510+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:35.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:35.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:35.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:35.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:35.581 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:35.582 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:35 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.395 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:36.395 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:36.395 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:36.396 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:36.397 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:36.398 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:36.399 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:36.400 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:36.400 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:36.400 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:36.400 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:36.400 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:36.400 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.401 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.401 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.401 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:36.401 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:36.401 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:36.402 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:36.402 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:36.402 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:36.402 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:36.402 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:36.402 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:36.403 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:36.403 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:36.403 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:17:36.403 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:36.403 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:36.403 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:36.404 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.405 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:36.406 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:36.407 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:36.408 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.409 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: Upgrade: It appears safe to stop mon.a 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[102491]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: Upgrade: It appears safe to stop mon.a 2024-09-13T22:17:36.410 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:36 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:36.613 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:36.614 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:36.615 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:36.616 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:36.617 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: Upgrade: It appears safe to stop mon.a 2024-09-13T22:17:36.618 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:36 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:39.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-09-13T22:17:39.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 829 B/s rd, 0 op/s 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[102491]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 829 B/s rd, 0 op/s 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:39 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-09-13T22:17:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 829 B/s rd, 0 op/s 2024-09-13T22:17:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:17:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:17:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:17:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:39 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:40.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[28528]: Upgrade: Updating mon.a 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[28528]: Deploying daemon mon.a on smithi114 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[28528]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[28528]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[102491]: Upgrade: Updating mon.a 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[102491]: Deploying daemon mon.a on smithi114 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[102491]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[102491]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:40 smithi114 ceph-mon[102491]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:40.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:40 smithi114 systemd[1]: Stopping Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:17:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:40 smithi121 ceph-mon[83513]: Upgrade: Updating mon.a 2024-09-13T22:17:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:40 smithi121 ceph-mon[83513]: Deploying daemon mon.a on smithi114 2024-09-13T22:17:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:40 smithi121 ceph-mon[83513]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-09-13T22:17:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:40 smithi121 ceph-mon[83513]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:40.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:41.723 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[102487]: 2024-09-13T22:17:41.418+0000 7f061e5c7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:17:41.724 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[102487]: 2024-09-13T22:17:41.418+0000 7f061e5c7640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T22:17:41.724 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:41 smithi114 podman[105312]: 2024-09-13 22:17:41.45880089 +0000 UTC m=+0.149582242 container stop c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T22:17:41.724 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:41 smithi114 podman[105312]: 2024-09-13 22:17:41.486357371 +0000 UTC m=+0.177138724 container died c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T22:17:42.028 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:17:41] "GET /metrics HTTP/1.1" 200 38029 "" "Prometheus/2.43.0" 2024-09-13T22:17:42.028 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:41 smithi114 podman[105312]: 2024-09-13 22:17:41.834482448 +0000 UTC m=+0.525263800 container cleanup c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T22:17:42.028 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:41 smithi114 bash[105312]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a 2024-09-13T22:17:42.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 podman[105376]: 2024-09-13 22:17:42.027513652 +0000 UTC m=+0.561710294 container remove c00c3478a2b07c82ac696048b019a495f7c4f9197913e3e23cc5c38648b27a00 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-ecab7be) 2024-09-13T22:17:42.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T22:17:42.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 systemd[1]: Stopped Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:17:42.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service: Consumed 1.368s CPU time. 2024-09-13T22:17:42.541 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 systemd[1]: Starting Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:17:42.541 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:17:42.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 podman[105557]: 2024-09-13 22:17:42.601978642 +0000 UTC m=+0.015964920 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:17:42.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 podman[105557]: 2024-09-13 22:17:42.728589524 +0000 UTC m=+0.142575801 container create 96aac05a92f368526c3157e8db5094629b5b16ae2a3a6fba6c92f6db8be184fe (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 1s ago 30m - - 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 5s ago 30m - - 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (16m) 1s ago 24m 26.2M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (2m) 5s ago 24m 81.3M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 1s ago 23m 59.1M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (3m) 5s ago 29m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (13m) 1s ago 31m 497M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:17:43.234 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 starting - - - 2048M 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (62s) 5s ago 29m 40.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (29m) 1s ago 29m 70.0M 2048M 17.2.0 e1d6a67b021e 92fe31687596 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (16m) 1s ago 25m 14.5M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (16m) 5s ago 25m 14.8M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (29m) 1s ago 29m 56.5M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (28m) 1s ago 28m 57.8M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (28m) 1s ago 28m 51.0M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (28m) 1s ago 28m 52.0M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (27m) 5s ago 27m 54.8M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (27m) 5s ago 27m 55.0M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (26m) 5s ago 26m 53.2M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (26m) 5s ago 26m 53.5M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (3m) 5s ago 25m 67.1M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:17:43.235 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (24m) 1s ago 24m 76.8M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:17:43.236 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (24m) 5s ago 24m 78.2M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:17:43.237 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 podman[105557]: 2024-09-13 22:17:42.956316145 +0000 UTC m=+0.370302420 container init 96aac05a92f368526c3157e8db5094629b5b16ae2a3a6fba6c92f6db8be184fe (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T22:17:43.237 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 podman[105557]: 2024-09-13 22:17:42.959660026 +0000 UTC m=+0.373646302 container start 96aac05a92f368526c3157e8db5094629b5b16ae2a3a6fba6c92f6db8be184fe (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-09-13T22:17:43.237 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:17:43.237 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T22:17:43.237 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: pidfile_write: ignore empty --pid-file 2024-09-13T22:17:43.237 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: load: jerasure load: lrc 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: RocksDB version: 7.9.2 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Git sha 0 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: DB SUMMARY 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: DB Session ID: 9EIVM02L7LARA27C1T22 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: MANIFEST file: MANIFEST-000080 size: 755 Bytes 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000088.sst 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000086.log size: 424688 ; 2024-09-13T22:17:43.238 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:17:43.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:17:43.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:17:43.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T22:17:43.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:17:43.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T22:17:43.239 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.env: 0x559fa61db940 2024-09-13T22:17:43.240 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T22:17:43.240 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.info_log: 0x559fa84316e0 2024-09-13T22:17:43.240 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:17:43.240 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.statistics: (nil) 2024-09-13T22:17:43.240 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.use_fsync: 0 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:17:43.241 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.db_log_dir: 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.wal_dir: 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:17:43.242 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.write_buffer_manager: 0x559fa84408c0 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.unordered_write: 0 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:17:43.243 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.row_cache: None 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.wal_filter: None 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.wal_compression: 0 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:17:43.244 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:17:43.245 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_open_files: -1 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:17:43.246 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Compression algorithms supported: 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kZSTD supported: 0 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:17:43.247 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000080 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.merge_operator: 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_filter: None 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559fa8431320) 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:17:43.248 INFO:journalctl@ceph.mon.a.smithi114.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_type: 0 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: data_block_index_type: 0 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_shortening: 1 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: checksum: 4 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: no_block_cache: 0 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache: 0x559fa84251f0 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_options: 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: capacity : 536870912 2024-09-13T22:17:43.249 INFO:journalctl@ceph.mon.a.smithi114.stdout: num_shard_bits : 4 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: strict_capacity_limit : 0 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_cache_compressed: (nil) 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: persistent_cache: (nil) 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_size: 4096 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_size_deviation: 10 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_restart_interval: 16 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: index_block_restart_interval: 1 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: metadata_block_size: 4096 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: partition_filters: 0 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: use_delta_encoding: 1 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: filter_policy: bloomfilter 2024-09-13T22:17:43.250 INFO:journalctl@ceph.mon.a.smithi114.stdout: whole_key_filtering: 1 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: verify_compression: 0 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: format_version: 5 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: enable_index_compression: 1 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: block_align: 0 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: max_auto_readahead_size: 262144 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: prepopulate_block_cache: 0 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: initial_auto_readahead_size: 8192 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression: NoCompression 2024-09-13T22:17:43.251 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.num_levels: 7 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:17:43.252 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:17:43.253 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:17:43.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:17:43.255 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:17:43.256 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.ttl: 2592000 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:17:43.257 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000080 succeeded,manifest_file_number is 80, next_file_number is 90, last_sequence is 37244, log_number is 86,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 86 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 86 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3b5bcef9-ab59-4baf-9176-b2df0666f9a5 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265862986512, "job": 1, "event": "recovery_started", "wal_files": [86]} 2024-09-13T22:17:43.258 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #86 mode 2 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265862988512, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 91, "file_size": 389647, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 37246, "largest_seqno": 37442, "table_properties": {"data_size": 387380, "index_size": 973, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2429, "raw_average_key_size": 24, "raw_value_size": 384816, "raw_average_value_size": 3848, "num_data_blocks": 42, "num_entries": 100, "num_filter_entries": 100, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726265862, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3b5bcef9-ab59-4baf-9176-b2df0666f9a5", "db_session_id": "9EIVM02L7LARA27C1T22", "orig_file_number": 91, "seqno_to_time_mapping": "N/A"}} 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265862988621, "job": 1, "event": "recovery_finished"} 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/version_set.cc:5047] Creating manifest 93 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000086.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559fa8452e00 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: DB pointer 0x559fa854e000 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** DB Stats ** 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:17:43.259 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: L0 1/0 380.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 274.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: L6 1/0 14.50 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Sum 2/0 14.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 274.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:17:43.260 INFO:journalctl@ceph.mon.a.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 274.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 274.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:17:43.261 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:17:43.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:17:43.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:17:43.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:17:43.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Cumulative compaction: 0.00 GB write, 84.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:17:43.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Interval compaction: 0.00 GB write, 84.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:17:43.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:17:43.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: Block cache BinnedLRUCache@0x559fa84251f0#2 capacity: 512.00 MB usage: 1.39 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T22:17:43.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2024-09-13T22:17:43.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:43.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:17:43.264 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: starting mon.a rank 0 at public addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] at bind addrs [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:43.264 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???) e3 preinit fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???).mds e1 new map 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???).mds e1 print_map 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: e1 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: legacy client fscid: -1 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: No filesystems configured 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:17:43.265 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:17:43.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:42 smithi114 ceph-mon[105598]: mon.a@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-13T22:17:43.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:43 smithi114 bash[105557]: 96aac05a92f368526c3157e8db5094629b5b16ae2a3a6fba6c92f6db8be184fe 2024-09-13T22:17:43.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:43 smithi114 systemd[1]: Started Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:17:43.636 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:17:43.636 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:17:43.636 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:17:43.636 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:17:43.636 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "mds": {}, 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-13T22:17:43.637 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T22:17:43.638 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:17:43.638 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:17:43.972 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:17:43.972 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:17:43.972 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:17:43.972 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi114", 2024-09-13T22:17:43.972 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:17:43.972 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "0/2 daemons upgraded", 2024-09-13T22:17:43.973 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T22:17:43.973 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:17:43.973 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:17:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='client.25702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='client.25708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='client.25714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='client.? 172.21.15.114:0/2931818318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: mon.a calling monitor election 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: monmap epoch 3 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: last_changed 2024-09-13T21:47:54.401708+0000 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: min_mon_release 17 (quincy) 2024-09-13T22:17:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: election_strategy: 1 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: fsmap 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: mgrmap e49: y(active, since 19s), standbys: x 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='' 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='' 2024-09-13T22:17:44.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='client.25702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='client.25708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='client.25714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/2931818318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:44.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: mon.a calling monitor election 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: monmap epoch 3 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: last_changed 2024-09-13T21:47:54.401708+0000 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: min_mon_release 17 (quincy) 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: election_strategy: 1 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: fsmap 2024-09-13T22:17:44.578 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: mgrmap e49: y(active, since 19s), standbys: x 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='' 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='' 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:44.579 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:44 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='client.25702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='client.25708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='client.25714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/2931818318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: mon.a calling monitor election 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: monmap epoch 3 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: last_changed 2024-09-13T21:47:54.401708+0000 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: min_mon_release 17 (quincy) 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: election_strategy: 1 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:17:44.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: fsmap 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: mgrmap e49: y(active, since 19s), standbys: x 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='' 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='' 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:44.873 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:44.874 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:44 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:17:45.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:17:45.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:45.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:45 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:17:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-09-13T22:17:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-09-13T22:17:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-09-13T22:17:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:45.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:45.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:45.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:45.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:45 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:46.124+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:17:46.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.757 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.757 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.758 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:46.759 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:46.760 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.761 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:46.761 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.761 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.761 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.761 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.761 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.761 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.762 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.763 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.764 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:17:46.765 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.766 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.767 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:46.768 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:17:46.769 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:46.770 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:46.770 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:46.770 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:46.770 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:46.770 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:46.770 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:46 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:46.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:46.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:46.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:46.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:46.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:46 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:48.928 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:48.773+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:48.929 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:48.930 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:48.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:48.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:48.933 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:49.565 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:17:49.565 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:49.565 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:49.566 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:49.567 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:49.567 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:49.567 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:49 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:49.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:17:49.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:49.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:49.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:49.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:49.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:49.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:49.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:49.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:49 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:50.484 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:50.484 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:50.484 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:50.484 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:50.484 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:50.485 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:50.486 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:50.487 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:50.487 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:50.487 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:50.487 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:50.488 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:50.489 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:50.490 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:50.491 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[105598]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[105598]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:17:50.492 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.493 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.494 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.495 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[28528]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:50.495 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[28528]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:50.495 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:50.495 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:50.495 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:50 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:50.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:50 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:50.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:50.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:50.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:17:50.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:50.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:50 smithi121 ceph-mon[83513]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:50 smithi121 ceph-mon[83513]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:50 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:50 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:50.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:50 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:51.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:17:51] "GET /metrics HTTP/1.1" 200 38029 "" "Prometheus/2.43.0" 2024-09-13T22:17:52.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:52 smithi114 ceph-mon[105598]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 803 B/s rd, 0 op/s 2024-09-13T22:17:52.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:52 smithi114 ceph-mon[28528]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 803 B/s rd, 0 op/s 2024-09-13T22:17:52.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:52 smithi121 ceph-mon[83513]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 803 B/s rd, 0 op/s 2024-09-13T22:17:53.705 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:53.538+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:53.705 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:53.706 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:53.707 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:17:53.708 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:53.709 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:53.710 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:53 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:54.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 769 B/s rd, 0 op/s 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 769 B/s rd, 0 op/s 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:54.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:54.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:54.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:54.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:54.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:54.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:54 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:54.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 769 B/s rd, 0 op/s 2024-09-13T22:17:54.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:54.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:54.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:54.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:54.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:54.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:54.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:54.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:54 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.573 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:17:55.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:55.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[105598]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 690 B/s rd, 0 op/s 2024-09-13T22:17:55.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:55.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:55.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[105598]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:55.580 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:55.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:55.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[28528]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 690 B/s rd, 0 op/s 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[28528]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:55.583 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:55 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:55 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:55.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:55.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:55.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:55 smithi121 ceph-mon[83513]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 690 B/s rd, 0 op/s 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:55 smithi121 ceph-mon[83513]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-09-13T22:17:55.625 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:55 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:55.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:55 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:55.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:55 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:55.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:55 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:17:57.380 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:57 smithi114 ceph-mon[105598]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 756 B/s rd, 0 op/s 2024-09-13T22:17:57.380 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:57 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:57.380 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:57 smithi114 ceph-mon[28528]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 756 B/s rd, 0 op/s 2024-09-13T22:17:57.380 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:57 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:57.491 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:57 smithi121 ceph-mon[83513]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 756 B/s rd, 0 op/s 2024-09-13T22:17:57.492 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:57 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:17:58.210 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:17:58.018+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:58.210 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:58.211 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:58.212 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:58.212 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:58.212 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:58.212 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:58.212 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:58.212 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:58.212 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:58.213 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:58.214 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:58.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:58.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:17:58.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:58.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:17:58.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:58.215 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:58.216 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:17:58 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:17:59.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:59.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.126 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.127 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:17:59 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.147 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:59.147 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:59.147 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:59.147 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:59.147 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:59.147 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:59.148 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:59.149 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:59.150 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:59.151 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.151 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.151 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.151 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:59.151 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:59.151 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:59.151 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.152 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:59.153 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:17:59.154 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:17:59.155 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:17:59.156 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:17:59.157 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:17:59.158 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.159 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:17:59.160 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:17:59 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[105598]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:00.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[105598]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:00.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[28528]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[28528]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.291 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:00 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:00 smithi121 ceph-mon[83513]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:00.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:00 smithi121 ceph-mon[83513]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:00.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:00 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:00 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:00 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:00 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:00.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:00 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[105598]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[105598]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[105598]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[105598]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[28528]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[28528]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[28528]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[28528]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.267 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:01 smithi121 ceph-mon[83513]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:01.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:01 smithi121 ceph-mon[83513]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:01 smithi121 ceph-mon[83513]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:01 smithi121 ceph-mon[83513]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:18:01.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:01 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:02.064 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:18:01] "GET /metrics HTTP/1.1" 200 38024 "" "Prometheus/2.43.0" 2024-09-13T22:18:03.205 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-mon[105598]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:18:03.205 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-mon[28528]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:18:03.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:03 smithi121 ceph-mon[83513]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:18:03.324+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:03.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:03.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:03.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:03.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:03.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:04.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:18:04.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:04.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:18:04.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T22:18:04.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:18:04.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:04.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:18:04.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:04 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T22:18:04.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:04 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:18:04.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:04 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:04.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:04 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:18:04.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:04 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T22:18:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:05 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:05.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:05.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:05.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:05.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:05.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:05.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:05.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:05 smithi114 ceph-mon[28528]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:05.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:05.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:05.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:05.335 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:05 smithi114 ceph-mon[105598]: Upgrade: It appears safe to stop mon.c 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:05 smithi114 ceph-mon[105598]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 904 B/s rd, 0 op/s 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:05.336 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.337 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.338 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.339 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:05 smithi114 ceph-mon[28528]: Upgrade: It appears safe to stop mon.c 2024-09-13T22:18:05.339 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:05 smithi114 ceph-mon[28528]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 904 B/s rd, 0 op/s 2024-09-13T22:18:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:05 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:05.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:05.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:18:05.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:05.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:05 smithi121 ceph-mon[83513]: Upgrade: It appears safe to stop mon.c 2024-09-13T22:18:05.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:05 smithi121 ceph-mon[83513]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 904 B/s rd, 0 op/s 2024-09-13T22:18:06.802 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: Upgrade: Updating mon.c 2024-09-13T22:18:06.802 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:06.802 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: Deploying daemon mon.c on smithi114 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 794 B/s rd, 0 op/s 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: Upgrade: Updating mon.c 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:06.803 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: Deploying daemon mon.c on smithi114 2024-09-13T22:18:06.804 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 794 B/s rd, 0 op/s 2024-09-13T22:18:06.804 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 ceph-mon[28528]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:06.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: Upgrade: Updating mon.c 2024-09-13T22:18:06.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: from='mgr.25675 ' entity='mgr.y' 2024-09-13T22:18:06.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:06.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:06.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:06.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: Deploying daemon mon.c on smithi114 2024-09-13T22:18:06.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 794 B/s rd, 0 op/s 2024-09-13T22:18:06.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:06 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:07.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:06 smithi114 systemd[1]: Stopping Ceph mon.c for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:18:07.514 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c[28524]: 2024-09-13T22:18:07.180+0000 7f4ecb714700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:18:07.514 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c[28524]: 2024-09-13T22:18:07.180+0000 7f4ecb714700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T22:18:07.514 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:07 smithi114 podman[108111]: 2024-09-13 22:18:07.22318233 +0000 UTC m=+0.173344545 container died 92fe3168759696038ddc4da7e23df6e10f4d90b8120487116bbe6949076eac74 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, distribution-scope=public, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=centos-stream, RELEASE=HEAD, GIT_CLEAN=True, ceph=True) 2024-09-13T22:18:07.514 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:18:07] ENGINE Bus STOPPING 2024-09-13T22:18:07.514 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:18:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:18:07.514 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:18:07] ENGINE Bus STOPPED 2024-09-13T22:18:07.514 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:18:07] ENGINE Bus STARTING 2024-09-13T22:18:07.515 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:18:07] ENGINE Serving on http://:::9283 2024-09-13T22:18:07.515 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:18:07] ENGINE Bus STARTED 2024-09-13T22:18:07.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:07 smithi114 podman[108111]: 2024-09-13 22:18:07.604179162 +0000 UTC m=+0.554341379 container cleanup 92fe3168759696038ddc4da7e23df6e10f4d90b8120487116bbe6949076eac74 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, ceph=True, io.buildah.version=1.19.8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:18:07.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:07 smithi114 bash[108111]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c 2024-09-13T22:18:08.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:07 smithi114 podman[108123]: 2024-09-13 22:18:07.872455806 +0000 UTC m=+0.648341209 container remove 92fe3168759696038ddc4da7e23df6e10f4d90b8120487116bbe6949076eac74 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, name=centos-stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, release=754) 2024-09-13T22:18:08.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T22:18:08.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 systemd[1]: Stopped Ceph mon.c for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:18:08.215 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.c.service: Consumed 18.433s CPU time. 2024-09-13T22:18:08.563 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 systemd[1]: Starting Ceph mon.c for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:18:08.564 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 podman[108267]: 2024-09-13 22:18:08.324905467 +0000 UTC m=+0.016124651 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:18:08.564 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 podman[108267]: 2024-09-13 22:18:08.423220219 +0000 UTC m=+0.114439394 container create 6670a4db3663d319563d861d032c4a638c4cf26f8dcbc9731c73ae85a4f3c934 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T22:18:08.820 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 podman[108267]: 2024-09-13 22:18:08.668025565 +0000 UTC m=+0.359244746 container init 6670a4db3663d319563d861d032c4a638c4cf26f8dcbc9731c73ae85a4f3c934 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:18:08.820 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 podman[108267]: 2024-09-13 22:18:08.671994347 +0000 UTC m=+0.363213525 container start 6670a4db3663d319563d861d032c4a638c4cf26f8dcbc9731c73ae85a4f3c934 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be) 2024-09-13T22:18:08.820 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:18:08.820 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T22:18:08.820 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: pidfile_write: ignore empty --pid-file 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: load: jerasure load: lrc 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: RocksDB version: 7.9.2 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Git sha 0 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: DB SUMMARY 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: DB Session ID: KFUKOFL7W77PUKBDP7PN 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: MANIFEST file: MANIFEST-000009 size: 4365 Bytes 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000078.sst 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000076.log size: 1103428 ; 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:18:08.821 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.env: 0x5578eeeba940 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.info_log: 0x5578f090b6e0 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:18:08.822 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.statistics: (nil) 2024-09-13T22:18:08.823 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.use_fsync: 0 2024-09-13T22:18:08.823 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.db_log_dir: 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.wal_dir: 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:18:08.824 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.write_buffer_manager: 0x5578f091a8c0 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:18:08.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.unordered_write: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.row_cache: None 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.wal_filter: None 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.wal_compression: 0 2024-09-13T22:18:08.826 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:18:08.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_open_files: -1 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:18:08.828 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Compression algorithms supported: 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kZSTD supported: 0 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T22:18:08.829 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.merge_operator: 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_filter: None 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5578f090b320) 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: index_type: 0 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: data_block_index_type: 0 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: index_shortening: 1 2024-09-13T22:18:08.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: checksum: 4 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: no_block_cache: 0 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache: 0x5578f08ff1f0 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache_options: 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: capacity : 536870912 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: num_shard_bits : 4 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: strict_capacity_limit : 0 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_cache_compressed: (nil) 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: persistent_cache: (nil) 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_size: 4096 2024-09-13T22:18:08.831 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_size_deviation: 10 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_restart_interval: 16 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: index_block_restart_interval: 1 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: metadata_block_size: 4096 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: partition_filters: 0 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: use_delta_encoding: 1 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: filter_policy: bloomfilter 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: whole_key_filtering: 1 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: verify_compression: 0 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: format_version: 5 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: enable_index_compression: 1 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: block_align: 0 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: max_auto_readahead_size: 262144 2024-09-13T22:18:08.832 INFO:journalctl@ceph.mon.c.smithi114.stdout: prepopulate_block_cache: 0 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: initial_auto_readahead_size: 8192 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression: NoCompression 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:18:08.833 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.num_levels: 7 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:18:08.834 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:18:08.835 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:18:08.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:18:08.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:18:08.838 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.ttl: 2592000 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:18:08.839 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 78.sst 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 80, last_sequence is 40884, log_number is 76,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 76 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fb947ab7-7802-451b-a8df-078f9aaada4c 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265888698828, "job": 1, "event": "recovery_started", "wal_files": [76]} 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #76 mode 2 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265888702400, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 81, "file_size": 712443, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 40881, "largest_seqno": 41619, "table_properties": {"data_size": 708350, "index_size": 1772, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1285, "raw_key_size": 9692, "raw_average_key_size": 19, "raw_value_size": 699726, "raw_average_value_size": 1407, "num_data_blocks": 75, "num_entries": 497, "num_filter_entries": 497, "num_deletions": 250, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726265888, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fb947ab7-7802-451b-a8df-078f9aaada4c", "db_session_id": "KFUKOFL7W77PUKBDP7PN", "orig_file_number": 81, "seqno_to_time_mapping": "N/A"}} 2024-09-13T22:18:08.840 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726265888702504, "job": 1, "event": "recovery_finished"} 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/version_set.cc:5047] Creating manifest 83 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000076.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5578f092ce00 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: DB pointer 0x5578f0a28000 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** DB Stats ** 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:18:08.841 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: L0 1/0 695.75 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 366.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: L6 1/0 12.62 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: Sum 2/0 13.30 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 366.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 366.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T22:18:08.842 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** Compaction Stats [default] ** 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 366.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:18:08.843 INFO:journalctl@ceph.mon.c.smithi114.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Cumulative compaction: 0.00 GB write, 82.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Interval compaction: 0.00 GB write, 82.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Block cache BinnedLRUCache@0x5578f08ff1f0#2 capacity: 512.00 MB usage: 3.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: starting mon.c rank 1 at public addrs [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] at bind addrs [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???) e3 preinit fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???).mds e1 new map 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???).mds e1 print_map 2024-09-13T22:18:08.844 INFO:journalctl@ceph.mon.c.smithi114.stdout: e1 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: legacy client fscid: -1 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout: No filesystems configured 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:18:08.845 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: mon.c@-1(???).paxosservice(auth 1..31) refresh upgraded, format 0 -> 3 2024-09-13T22:18:08.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 bash[108267]: 6670a4db3663d319563d861d032c4a638c4cf26f8dcbc9731c73ae85a4f3c934 2024-09-13T22:18:08.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 systemd[1]: Started Ceph mon.c for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:18:08.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:18:08.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:18:08.846 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:18:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:18:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:18:09.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:08 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:18:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:08 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:18:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:08 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:18:09.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:08 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:18:12.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:18:11] "GET /metrics HTTP/1.1" 200 38024 "" "Prometheus/2.43.0" 2024-09-13T22:18:14.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:18:13.715+0000 7fce0b9cb640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: mon.a calling monitor election 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: mon.c calling monitor election 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: monmap epoch 4 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: last_changed 2024-09-13T22:18:08.822343+0000 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:18:14.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: min_mon_release 19 (squid) 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: election_strategy: 1 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: fsmap 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: mgrmap e49: y(active, since 48s), standbys: x 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:14.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: mon.a calling monitor election 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: mon.c calling monitor election 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T22:18:14.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: monmap epoch 4 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: last_changed 2024-09-13T22:18:08.822343+0000 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: min_mon_release 19 (squid) 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: election_strategy: 1 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: fsmap 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: mgrmap e49: y(active, since 48s), standbys: x 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:18:14.078 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:14.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:14.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:14.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:14.079 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:13 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:14.342 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 15s ago 30m - - 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 16s ago 30m - - 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (17m) 15s ago 25m 26.2M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (2m) 16s ago 25m 81.3M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 15s ago 24m 60.7M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (3m) 16s ago 30m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (13m) 15s ago 31m 500M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (32s) 15s ago 31m 40.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 96aac05a92f3 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (94s) 16s ago 30m 39.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:18:15.012 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 starting - - - 2048M 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (17m) 15s ago 26m 14.9M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (16m) 16s ago 26m 15.4M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (29m) 15s ago 29m 56.5M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (29m) 15s ago 29m 57.8M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (29m) 15s ago 29m 51.0M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (28m) 15s ago 28m 52.0M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (28m) 16s ago 28m 54.8M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (27m) 16s ago 27m 55.0M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:18:15.013 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (27m) 16s ago 27m 53.3M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:18:15.016 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (27m) 16s ago 27m 53.5M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:18:15.016 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (4m) 16s ago 26m 67.1M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:18:15.016 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (24m) 15s ago 24m 76.8M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:18:15.016 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (24m) 16s ago 24m 78.3M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:18:15.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-09-13T22:18:15.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:15.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:18:15.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: Reconfiguring mon.a (monmap changed)... 2024-09-13T22:18:15.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[105598]: Reconfiguring daemon mon.a on smithi114 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: Reconfiguring mon.a (monmap changed)... 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:14 smithi114 ceph-mon[108287]: Reconfiguring daemon mon.a on smithi114 2024-09-13T22:18:15.423 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:18:15.423 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:18:15.423 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T22:18:15.423 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:18:15.423 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:18:15.423 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:18:15.423 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:18:15.424 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mon.a calling monitor election 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mon.c calling monitor election 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: monmap epoch 4 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: last_changed 2024-09-13T22:18:08.822343+0000 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: min_mon_release 19 (squid) 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: election_strategy: 1 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: fsmap 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mgrmap e49: y(active, since 48s), standbys: x 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:15.623 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: Reconfiguring mon.a (monmap changed)... 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:15.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: Reconfiguring daemon mon.a on smithi114 2024-09-13T22:18:15.774 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:18:15.774 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:18:15.774 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:18:15.775 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi114", 2024-09-13T22:18:15.775 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:18:15.775 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "1/2 daemons upgraded", 2024-09-13T22:18:15.775 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T22:18:15.775 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:18:15.775 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:18:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: mon.b calling monitor election 2024-09-13T22:18:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: mon.a calling monitor election 2024-09-13T22:18:16.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: mon.c calling monitor election 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: monmap epoch 4 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: last_changed 2024-09-13T22:18:08.822343+0000 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: min_mon_release 19 (squid) 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: election_strategy: 1 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: fsmap 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: mgrmap e49: y(active, since 50s), standbys: x 2024-09-13T22:18:16.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: from='client.? 172.21.15.114:0/4163482509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:16.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: mon.b calling monitor election 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: mon.a calling monitor election 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: mon.c calling monitor election 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: monmap epoch 4 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: last_changed 2024-09-13T22:18:08.822343+0000 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: min_mon_release 19 (squid) 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: election_strategy: 1 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:18:16.077 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: fsmap 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: mgrmap e49: y(active, since 50s), standbys: x 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/4163482509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:18:16.078 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:15 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:16.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mon.b calling monitor election 2024-09-13T22:18:16.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mon.a calling monitor election 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mon.c calling monitor election 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: monmap epoch 4 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: last_changed 2024-09-13T22:18:08.822343+0000 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: created 2024-09-13T21:46:14.012833+0000 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: min_mon_release 19 (squid) 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: election_strategy: 1 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: 0: [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] mon.a 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: 1: [v2:172.21.15.114:3301/0,v1:172.21.15.114:6790/0] mon.c 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: 2: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.b 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: fsmap 2024-09-13T22:18:16.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: mgrmap e49: y(active, since 50s), standbys: x 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/4163482509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:18:16.373 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:17.254 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:16 smithi121 ceph-mon[83513]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T22:18:17.254 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:16 smithi121 ceph-mon[83513]: Reconfiguring daemon mgr.y on smithi114 2024-09-13T22:18:17.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:16 smithi121 ceph-mon[83513]: from='client.64125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:17.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:16 smithi121 ceph-mon[83513]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:17.255 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:16 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[105598]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[105598]: Reconfiguring daemon mgr.y on smithi114 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[105598]: from='client.64125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[105598]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[108287]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[108287]: Reconfiguring daemon mgr.y on smithi114 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[108287]: from='client.64125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:17.285 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[108287]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:17.286 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:16 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:18.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:18.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[105598]: Reconfiguring mon.c (monmap changed)... 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[105598]: Reconfiguring daemon mon.c on smithi114 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[108287]: Reconfiguring mon.c (monmap changed)... 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:18.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:18.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:18.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:17 smithi114 ceph-mon[108287]: Reconfiguring daemon mon.c on smithi114 2024-09-13T22:18:18.264 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:18.264 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:18.264 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:17 smithi121 ceph-mon[83513]: Reconfiguring mon.c (monmap changed)... 2024-09-13T22:18:18.264 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:18.264 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:18.264 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:18.264 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:17 smithi121 ceph-mon[83513]: Reconfiguring daemon mon.c on smithi114 2024-09-13T22:18:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:18:19.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:18 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:19.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-13T22:18:19.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:19.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:18:19.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:18 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 793 B/s rd, 0 op/s 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.0 on smithi114 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:20.437 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 793 B/s rd, 0 op/s 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.0 on smithi114 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:20.438 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:20 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 793 B/s rd, 0 op/s 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.0 on smithi114 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:20 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:21.302 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:21.302 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:21.302 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[105598]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T22:18:21.302 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:18:21.302 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:21.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.1 on smithi114 2024-09-13T22:18:21.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:21.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:21.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[108287]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T22:18:21.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:18:21.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:21.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.1 on smithi114 2024-09-13T22:18:21.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:21 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:21.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:21 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:21.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:21 smithi121 ceph-mon[83513]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T22:18:21.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:21 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:18:21.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:21 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:21.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:21 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.1 on smithi114 2024-09-13T22:18:22.020 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:18:21] "GET /metrics HTTP/1.1" 200 38031 "" "Prometheus/2.43.0" 2024-09-13T22:18:22.288 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:22 smithi114 ceph-mon[105598]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:18:22.288 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:22 smithi114 ceph-mon[108287]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:18:22.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:22 smithi121 ceph-mon[83513]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-09-13T22:18:24.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[108287]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:24.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:24.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:24.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:18:24.206 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:24.207 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[105598]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:24.207 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:24.207 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:24.207 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:18:24.207 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:24 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:24.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:24 smithi121 ceph-mon[83513]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-09-13T22:18:24.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:24 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:24.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:24 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:24.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:24 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:18:24.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:24 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:25.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:25 smithi114 ceph-mon[108287]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T22:18:25.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:25 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.2 on smithi114 2024-09-13T22:18:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:25 smithi114 ceph-mon[105598]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T22:18:25.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:25 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.2 on smithi114 2024-09-13T22:18:25.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:25 smithi121 ceph-mon[83513]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T22:18:25.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:25 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.2 on smithi114 2024-09-13T22:18:26.295 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:26 smithi114 ceph-mon[105598]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 766 B/s rd, 0 op/s 2024-09-13T22:18:26.296 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:26 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:26.296 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:26 smithi114 ceph-mon[108287]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 766 B/s rd, 0 op/s 2024-09-13T22:18:26.296 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:26 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:26 smithi121 ceph-mon[83513]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 766 B/s rd, 0 op/s 2024-09-13T22:18:26.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:27.230 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:27.231 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.231 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.231 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.231 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:27.231 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.231 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.231 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:27 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:27 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:27 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:27 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:27.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:27 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:28.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:28 smithi114 ceph-mon[105598]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:28.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:28 smithi114 ceph-mon[105598]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:18:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:28 smithi114 ceph-mon[108287]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:28.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:28 smithi114 ceph-mon[108287]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:18:28.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:28 smithi121 ceph-mon[83513]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:28.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:28 smithi121 ceph-mon[83513]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:18:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[105598]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T22:18:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:18:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[108287]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T22:18:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:18:29.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:29 smithi121 ceph-mon[83513]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T22:18:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:18:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:30.406 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.3 on smithi114 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[108287]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.3 on smithi114 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[105598]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:18:30.407 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:30 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:30 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.3 on smithi114 2024-09-13T22:18:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:30 smithi121 ceph-mon[83513]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:30 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:30 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:30 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi114.vrzgda", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:18:30.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:30 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:31.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: Reconfiguring rgw.foo.smithi114.vrzgda (monmap changed)... 2024-09-13T22:18:31.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: Reconfiguring daemon rgw.foo.smithi114.vrzgda on smithi114 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 552 B/s rd, 0 op/s 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: Reconfiguring rgw.foo.smithi114.vrzgda (monmap changed)... 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: Reconfiguring daemon rgw.foo.smithi114.vrzgda on smithi114 2024-09-13T22:18:31.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 552 B/s rd, 0 op/s 2024-09-13T22:18:31.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:31.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:31.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:31.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:31.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:31.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: Reconfiguring rgw.foo.smithi114.vrzgda (monmap changed)... 2024-09-13T22:18:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: Reconfiguring daemon rgw.foo.smithi114.vrzgda on smithi114 2024-09-13T22:18:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 552 B/s rd, 0 op/s 2024-09-13T22:18:31.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:31.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:31.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:31.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:31.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:31.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:31 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:31.922 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:18:31] "GET /metrics HTTP/1.1" 200 38035 "" "Prometheus/2.43.0" 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:18:32.740+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:33.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:33.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.824 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[105598]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:33.825 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:33.826 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:33.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:33.827 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:33.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[108287]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:33.827 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.827 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.828 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.829 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:33.830 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.831 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.832 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[105598]: Reconfiguring mon.b (monmap changed)... 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:33.833 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[105598]: Reconfiguring daemon mon.b on smithi121 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:33.834 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.835 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[108287]: Reconfiguring mon.b (monmap changed)... 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:33.836 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:33.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:33.837 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:33 smithi114 ceph-mon[108287]: Reconfiguring daemon mon.b on smithi121 2024-09-13T22:18:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:33 smithi121 ceph-mon[83513]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:33.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:33 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.872 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.873 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:33.874 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:33.875 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:33.876 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:33 smithi121 ceph-mon[83513]: Reconfiguring mon.b (monmap changed)... 2024-09-13T22:18:33.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:33 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:18:33.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:33 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:18:33.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:33 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:33.877 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:33 smithi121 ceph-mon[83513]: Reconfiguring daemon mon.b on smithi121 2024-09-13T22:18:35.501 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:35 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:35.502 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:35 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:35.502 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:35 smithi121 ceph-mon[83513]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T22:18:35.502 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:35 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:18:35.502 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:35 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:18:35.502 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:35 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:35.502 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:35 smithi121 ceph-mon[83513]: Reconfiguring daemon mgr.x on smithi121 2024-09-13T22:18:35.504 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:35.504 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:35.504 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[105598]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T22:18:35.504 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:18:35.504 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:18:35.504 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:35.504 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[105598]: Reconfiguring daemon mgr.x on smithi121 2024-09-13T22:18:35.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:35.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:35.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[108287]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T22:18:35.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:18:35.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:18:35.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:35.505 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:35 smithi114 ceph-mon[108287]: Reconfiguring daemon mgr.x on smithi121 2024-09-13T22:18:36.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:36 smithi121 ceph-mon[83513]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:36.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:36.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:36.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:18:36.238 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:36.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[105598]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:36.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:36.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:36.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:18:36.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[108287]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:18:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:18:36.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:36 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:37 smithi121 ceph-mon[83513]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T22:18:37.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:37 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.4 on smithi121 2024-09-13T22:18:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:37 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:37 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:37 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:37 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:18:37.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:37 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[105598]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T22:18:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.4 on smithi121 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[108287]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.4 on smithi121 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:18:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:37 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:38.466 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:38 smithi121 ceph-mon[83513]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:38.466 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:38 smithi121 ceph-mon[83513]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T22:18:38.466 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:38 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.5 on smithi121 2024-09-13T22:18:38.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:38 smithi114 ceph-mon[105598]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:38.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:38 smithi114 ceph-mon[105598]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T22:18:38.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:38 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.5 on smithi121 2024-09-13T22:18:38.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:38 smithi114 ceph-mon[108287]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:38.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:38 smithi114 ceph-mon[108287]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T22:18:38.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:38 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.5 on smithi121 2024-09-13T22:18:39.643 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:18:39.643 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.643 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.643 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T22:18:39.643 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:18:39.643 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:39.643 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.6 on smithi121 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:39.644 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.6 on smithi121 2024-09-13T22:18:39.645 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.645 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.645 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-13T22:18:39.645 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:39.645 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.768 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:18:39.768 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.6 on smithi121 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:39.769 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:40.850 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 707 B/s rd, 0 op/s 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:18:40.851 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 707 B/s rd, 0 op/s 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:18:41.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 707 B/s rd, 0 op/s 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:18:41.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:41.988 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:41 smithi121 ceph-mon[83513]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T22:18:41.988 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:41 smithi121 ceph-mon[83513]: Reconfiguring daemon osd.7 on smithi121 2024-09-13T22:18:42.060 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:41 smithi114 ceph-mon[108287]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T22:18:42.061 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:41 smithi114 ceph-mon[108287]: Reconfiguring daemon osd.7 on smithi121 2024-09-13T22:18:42.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:41 smithi114 ceph-mon[105598]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T22:18:42.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:41 smithi114 ceph-mon[105598]: Reconfiguring daemon osd.7 on smithi121 2024-09-13T22:18:42.061 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:18:41] "GET /metrics HTTP/1.1" 200 38035 "" "Prometheus/2.43.0" 2024-09-13T22:18:42.766 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:42 smithi114 ceph-mon[105598]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:42.767 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:42 smithi114 ceph-mon[108287]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:42.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:42 smithi121 ceph-mon[83513]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:44.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[108287]: Reconfiguring rgw.foo.smithi121.ihcnui (monmap changed)... 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[108287]: Reconfiguring daemon rgw.foo.smithi121.ihcnui on smithi121 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[108287]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[105598]: Reconfiguring rgw.foo.smithi121.ihcnui (monmap changed)... 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:44.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[105598]: Reconfiguring daemon rgw.foo.smithi121.ihcnui on smithi121 2024-09-13T22:18:44.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:44 smithi114 ceph-mon[105598]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:44.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:44 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:44.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:44 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:44.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:44 smithi121 ceph-mon[83513]: Reconfiguring rgw.foo.smithi121.ihcnui (monmap changed)... 2024-09-13T22:18:44.827 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:44 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi121.ihcnui", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:18:44.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:44 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:44.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:44 smithi121 ceph-mon[83513]: Reconfiguring daemon rgw.foo.smithi121.ihcnui on smithi121 2024-09-13T22:18:44.828 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:44 smithi121 ceph-mon[83513]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all mon 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.881 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all crash 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all mds 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:18:45.882 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all nfs 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.883 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.884 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all grafana 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all loki 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.885 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all promtail 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Finalizing container_image settings 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:18:45.886 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.887 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: Upgrade: Complete! 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:45.888 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:46.048 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.048 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.048 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:18:46.048 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all mon 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all crash 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.049 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all mds 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.050 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:18:46.051 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.051 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.051 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all nfs 2024-09-13T22:18:46.051 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.051 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.051 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:18:46.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all mon 2024-09-13T22:18:46.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.052 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all crash 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all mds 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.053 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all nfs 2024-09-13T22:18:46.054 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.055 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all grafana 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all loki 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all promtail 2024-09-13T22:18:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Finalizing container_image settings 2024-09-13T22:18:46.057 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:18:46.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:18:46.057 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all node-exporter 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all prometheus 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all alertmanager 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.058 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all grafana 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all loki 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all promtail 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T22:18:46.059 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Finalizing container_image settings 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:18:46.060 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.061 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:18:46.062 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.063 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: Upgrade: Complete! 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:18:46.064 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.065 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: Upgrade: Complete! 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:46.066 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:47.060 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:18:47.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[105598]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-09-13T22:18:47.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[105598]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-09-13T22:18:47.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:47.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:47.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[105598]: from='client.64131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:47.135 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:47.135 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[108287]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-09-13T22:18:47.136 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[108287]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-09-13T22:18:47.136 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:47.136 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:47.136 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[108287]: from='client.64131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:47.136 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:46 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:46 smithi121 ceph-mon[83513]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-09-13T22:18:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:46 smithi121 ceph-mon[83513]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-09-13T22:18:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:46 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:46 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:46 smithi121 ceph-mon[83513]: from='client.64131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:46 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:47.757 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:18:47.523+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:47.757 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:47.757 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:47.758 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:47.758 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:47.758 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:47.758 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:47.758 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:47.758 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:47.759 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:47.759 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:47.759 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:47.759 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:47.759 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:47.760 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:47.760 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:47.760 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:47.760 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:47.760 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:47.761 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:47.761 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:47.761 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:47.761 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:47.761 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:47.762 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:47.762 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:18:47.762 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:47.762 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:47.762 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:47.762 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:18:47.763 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:47.763 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:18:47.763 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:47.763 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:18:47.763 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:47.764 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:18:47.764 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:47.764 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:18:47.764 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:47.764 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:47.765 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:47.765 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:47.765 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:47.765 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:47.765 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:47.765 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:47.766 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:47.766 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:47.766 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:47.766 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:47.766 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:47.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:47.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:47.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:47.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:47.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:47.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:47.767 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:47.768 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:18:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:18:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:18:49.045 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[105598]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:49.045 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:49.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:49.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:49.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:49.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:49.045 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:49.046 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:49.047 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[108287]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.048 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:49.049 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:49.050 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.051 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.052 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:49.053 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:49.054 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[105598]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[105598]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:49.055 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:49.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[108287]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:49.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[108287]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-13T22:18:49.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:49.056 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:48 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:48 smithi121 ceph-mon[83513]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:48 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:49.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:18:49.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:49.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:49.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:49.377 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:49.377 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:49.377 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:49.377 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:49.377 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:49.377 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:48 smithi121 ceph-mon[83513]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:18:49.378 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:48 smithi121 ceph-mon[83513]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-13T22:18:49.378 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:48 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:18:49.378 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:48 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:18:49.496 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:18:49.496 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 9s ago 31m - - 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 10s ago 31m - - 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (17m) 9s ago 26m 26.3M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (3m) 10s ago 25m 80.1M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 9s ago 24m 62.9M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (4m) 10s ago 30m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (14m) 9s ago 32m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (66s) 9s ago 32m 46.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 96aac05a92f3 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (2m) 10s ago 30m 42.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (41s) 9s ago 31m 30.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 6670a4db3663 2024-09-13T22:18:49.497 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (18m) 9s ago 26m 14.8M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (17m) 10s ago 26m 14.9M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (30m) 9s ago 30m 56.9M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (30m) 9s ago 30m 57.5M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (29m) 9s ago 29m 51.1M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (29m) 9s ago 29m 52.3M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (28m) 10s ago 28m 54.7M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (28m) 10s ago 28m 54.9M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (28m) 10s ago 28m 53.3M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (27m) 10s ago 27m 53.5M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (4m) 10s ago 26m 67.2M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (25m) 9s ago 25m 77.4M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:18:49.498 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (25m) 10s ago 25m 78.5M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:18:50.660+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:50.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:50.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:18:50.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:50.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:50.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:51.048 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-09-13T22:18:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-mon[105598]: from='client.64137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:51.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-mon[105598]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-13T22:18:51.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-mon[108287]: from='client.64137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:51.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:50 smithi114 ceph-mon[108287]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-13T22:18:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:50 smithi121 ceph-mon[83513]: from='client.64137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:18:51.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:50 smithi121 ceph-mon[83513]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-13T22:18:52.012 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:18:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:18:51] "GET /metrics HTTP/1.1" 200 38035 "" "Prometheus/2.43.0" 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:51 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:52.013 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:52.014 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:52.015 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:52.016 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:51 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:52.017 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:52.018 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:52.018 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:52.018 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:52.018 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.018 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.018 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.019 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:18:52.020 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:52.021 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.022 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:51 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:52.121 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:52.122 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:18:52.123 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:18:52.124 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:18:52.126 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:18:53.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:52 smithi114 ceph-mon[105598]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:18:53.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:52 smithi114 ceph-mon[108287]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:18:53.175 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:18:53.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:52 smithi121 ceph-mon[83513]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:18:53.962 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-09-13T22:18:54.263 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:53 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/1103958991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:54.263 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:53 smithi114 ceph-mon[108287]: from='client.? 172.21.15.114:0/1103958991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:54.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:53 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/1103958991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:55.168 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:54 smithi114 ceph-mon[108287]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:55.168 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:54 smithi114 ceph-mon[105598]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:55.322 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:54 smithi121 ceph-mon[83513]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:18:55.977 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T22:18:56.211 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:55 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:56.212 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:55 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:56.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:55 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:18:56.568 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-09-13T22:18:57.226 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:56 smithi114 ceph-mon[105598]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-09-13T22:18:57.226 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:56 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/2346767618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:57.226 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:56 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:57.227 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:56 smithi114 ceph-mon[108287]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-09-13T22:18:57.227 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:56 smithi114 ceph-mon[108287]: from='client.? 172.21.15.114:0/2346767618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:57.227 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:56 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:56 smithi121 ceph-mon[83513]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-09-13T22:18:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:56 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/2346767618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:18:57.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:56 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:18:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:58 smithi121 ceph-mon[83513]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.184 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[108287]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:59.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.185 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.185 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[105598]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:18:59.185 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.185 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.185 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:18:59.906 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:19:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:59 smithi121 ceph-mon[83513]: from='client.64155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:00.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:59 smithi121 ceph-mon[83513]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-09-13T22:19:00.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:59 smithi121 ceph-mon[83513]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:00.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:18:59 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[108287]: from='client.64155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[108287]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[108287]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[105598]: from='client.64155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[105598]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-09-13T22:19:00.142 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[105598]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:00.143 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.143 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.143 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:18:59 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:00.491 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T22:19:01.035 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:00 smithi114 ceph-mon[105598]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-13T22:19:01.035 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:00 smithi114 ceph-mon[105598]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-09-13T22:19:01.035 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:00 smithi114 ceph-mon[108287]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-13T22:19:01.035 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:00 smithi114 ceph-mon[108287]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-09-13T22:19:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:00 smithi121 ceph-mon[83513]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-13T22:19:01.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:00 smithi121 ceph-mon[83513]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-09-13T22:19:01.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:19:01] "GET /metrics HTTP/1.1" 200 38031 "" "Prometheus/2.43.0" 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": null, 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": false, 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "", 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": null, 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "", 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:19:02.426 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:19:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:02 smithi114 ceph-mon[105598]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:03.169 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:02 smithi114 ceph-mon[108287]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:03.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:02 smithi121 ceph-mon[83513]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:03.620 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T22:19:04.261 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:03 smithi114 ceph-mon[105598]: from='client.74131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:04.262 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:03 smithi114 ceph-mon[108287]: from='client.74131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:04.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:03 smithi121 ceph-mon[83513]: from='client.74131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:05.220 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:04 smithi114 ceph-mon[105598]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-09-13T22:19:05.220 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:04 smithi114 ceph-mon[108287]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-09-13T22:19:05.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:04 smithi121 ceph-mon[83513]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-09-13T22:19:06.812 INFO:teuthology.orchestra.run.smithi114.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-13T22:19:06.812 INFO:teuthology.orchestra.run.smithi114.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:19:06.812 INFO:teuthology.orchestra.run.smithi114.stdout: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:19:07.005 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:06 smithi114 ceph-mon[105598]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 726 B/s rd, 0 op/s 2024-09-13T22:19:07.005 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:06 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:07.005 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:06 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/562747889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:19:07.005 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:06 smithi114 ceph-mon[108287]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 726 B/s rd, 0 op/s 2024-09-13T22:19:07.006 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:06 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:07.006 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:06 smithi114 ceph-mon[108287]: from='client.? 172.21.15.114:0/562747889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:19:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:06 smithi121 ceph-mon[83513]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 726 B/s rd, 0 op/s 2024-09-13T22:19:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:06 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:07.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:06 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/562747889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:19:07.609 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-09-13T22:19:09.188 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:08 smithi121 ceph-mon[83513]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:09.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:08 smithi114 ceph-mon[105598]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:09.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:08 smithi114 ceph-mon[108287]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:11.075 INFO:teuthology.orchestra.run.smithi114.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:10 smithi114 ceph-mon[105598]: from='client.74143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:10 smithi114 ceph-mon[105598]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:19:11.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:10 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:10 smithi114 ceph-mon[108287]: from='client.74143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:10 smithi114 ceph-mon[108287]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:19:11.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:10 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:11.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:10 smithi121 ceph-mon[83513]: from='client.74143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:11.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:10 smithi121 ceph-mon[83513]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-13T22:19:11.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:10 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:11.761 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T22:19:11.990 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:19:11] "GET /metrics HTTP/1.1" 200 38032 "" "Prometheus/2.43.0" 2024-09-13T22:19:12.302 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:12.302 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 904 B/s rd, 0 op/s 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 549 B/s rd, 0 op/s 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:12.303 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 904 B/s rd, 0 op/s 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 549 B/s rd, 0 op/s 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:12.304 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:12 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:12.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:12.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:12.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 904 B/s rd, 0 op/s 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 549 B/s rd, 0 op/s 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:12.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:12 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:14.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:19:13.716+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:14.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:14.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:14.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:14.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:14.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:14.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:19:14.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:14.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:14.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:14.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:14.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:14.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:14.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:14.370 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:14 smithi114 ceph-mon[105598]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:14.370 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:14 smithi114 ceph-mon[108287]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:14.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:14 smithi121 ceph-mon[83513]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:14.885 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:19:15.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:15 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:15.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:15.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:15.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:15.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:15.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:15 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:15.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:15.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:15.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:15.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:15 smithi114 ceph-mon[108287]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:19:15.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:15.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.333 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:15 smithi114 ceph-mon[105598]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:15.551 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:15 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:15.551 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:15.551 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:15.551 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:15.551 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:15.551 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.551 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:15.552 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:15.553 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:15.554 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:15.555 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:15 smithi121 ceph-mon[83513]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:15.556 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:19:15.556 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 15s ago 31m - - 2024-09-13T22:19:15.556 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 16s ago 31m - - 2024-09-13T22:19:15.556 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (18m) 15s ago 26m 26.3M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:19:15.556 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (3m) 16s ago 26m 80.1M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:19:15.556 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 15s ago 25m 62.9M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (4m) 16s ago 31m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (14m) 15s ago 32m 506M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (92s) 15s ago 32m 49.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 96aac05a92f3 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (2m) 16s ago 31m 44.2M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (67s) 15s ago 31m 33.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 6670a4db3663 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (18m) 15s ago 27m 15.3M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (17m) 16s ago 27m 14.9M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (30m) 15s ago 30m 56.9M 4096M 17.2.0 e1d6a67b021e 4d81e907e603 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (30m) 15s ago 30m 57.6M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (30m) 15s ago 30m 51.1M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (29m) 15s ago 29m 52.3M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (29m) 16s ago 29m 54.7M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (28m) 16s ago 28m 54.9M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:19:15.557 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (28m) 16s ago 28m 53.3M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:19:15.558 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (28m) 16s ago 28m 53.6M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:19:15.558 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (5m) 16s ago 27m 67.2M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:19:15.558 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (25m) 15s ago 25m 77.4M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:19:15.558 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (25m) 16s ago 25m 78.6M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:15.971 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:19:15.972 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:19:16.334 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:19:16.334 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:19:16.334 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:19:16.334 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-09-13T22:19:16.334 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:19:16.335 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "0/8 daemons upgraded", 2024-09-13T22:19:16.335 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:19:16.335 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:19:16.335 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='client.74149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='client.64185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.457 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/1759140393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='client.74149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='client.64185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.458 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.459 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.459 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.459 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.459 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.459 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:16.459 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:16 smithi114 ceph-mon[108287]: from='client.? 172.21.15.114:0/1759140393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='client.74149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='client.64185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:16.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:16.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:16 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/1759140393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:17.502 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: from='client.74161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:17.502 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:19:17.502 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all mgr 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all mon 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all crash 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: Upgrade: osd.0 is safe to restart 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: from='client.74161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:19:17.503 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all mgr 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all mon 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all crash 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: Upgrade: osd.0 is safe to restart 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:19:17.504 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: from='client.74161 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: Upgrade: Target is version 19.1.1-324-gecab7be5 (squid) 2024-09-13T22:19:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all mgr 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all mon 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all crash 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: Upgrade: osd.0 is safe to restart 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:19:17.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:18.484 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[105598]: from='client.64203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:18.484 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[105598]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:18.484 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[105598]: Upgrade: Updating osd.0 2024-09-13T22:19:18.484 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[105598]: Deploying daemon osd.0 on smithi114 2024-09-13T22:19:18.484 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[108287]: from='client.64203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:18.485 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[108287]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:18.485 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[108287]: Upgrade: Updating osd.0 2024-09-13T22:19:18.485 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:18 smithi114 ceph-mon[108287]: Deploying daemon osd.0 on smithi114 2024-09-13T22:19:18.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:18 smithi121 ceph-mon[83513]: from='client.64203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:18.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:18 smithi121 ceph-mon[83513]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:18.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:18 smithi121 ceph-mon[83513]: Upgrade: Updating osd.0 2024-09-13T22:19:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:18 smithi121 ceph-mon[83513]: Deploying daemon osd.0 on smithi114 2024-09-13T22:19:20.069 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.069 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.069 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[105598]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:20.069 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[105598]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:19:20.069 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.069 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.070 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.070 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[108287]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:20.070 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[108287]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:19:20.070 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:19 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:19 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:19 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:19 smithi121 ceph-mon[83513]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:19 smithi121 ceph-mon[83513]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:19:20.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:19 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:20.324 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:20 smithi114 systemd[1]: Stopping Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:19:20.824 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[31821]: 2024-09-13T22:19:20.327+0000 7f4a59249700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:19:20.824 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[31821]: 2024-09-13T22:19:20.327+0000 7f4a59249700 -1 osd.0 113 *** Got signal Terminated *** 2024-09-13T22:19:20.824 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[31821]: 2024-09-13T22:19:20.327+0000 7f4a59249700 -1 osd.0 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:19:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[105598]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:21.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[105598]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 0 op/s 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[105598]: osd.0 marked itself down and dead 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[108287]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[108287]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 0 op/s 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:20 smithi114 ceph-mon[108287]: osd.0 marked itself down and dead 2024-09-13T22:19:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:20 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:20 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:20 smithi121 ceph-mon[83513]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:19:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:20 smithi121 ceph-mon[83513]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 0 op/s 2024-09-13T22:19:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:20 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:21.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:20 smithi121 ceph-mon[83513]: osd.0 marked itself down and dead 2024-09-13T22:19:21.679 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:21 smithi114 podman[113444]: 2024-09-13 22:19:21.38987687 +0000 UTC m=+1.176054527 container died 4d81e907e6037aa8a65757a7505dfaade574d84f40fcbbfde5f0eba817b5c734 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , distribution-scope=public, vcs-type=git, io.openshift.tags=base centos centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, architecture=x86_64) 2024-09-13T22:19:22.062 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:21 smithi121 ceph-mon[83513]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:19:22.063 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:21 smithi121 ceph-mon[83513]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T22:19:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:21 smithi114 ceph-mon[108287]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:19:22.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:21 smithi114 ceph-mon[108287]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T22:19:22.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:21 smithi114 ceph-mon[105598]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:19:22.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:21 smithi114 ceph-mon[105598]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T22:19:22.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:19:21] "GET /metrics HTTP/1.1" 200 38032 "" "Prometheus/2.43.0" 2024-09-13T22:19:22.076 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:21 smithi114 podman[113444]: 2024-09-13 22:19:21.852964478 +0000 UTC m=+1.639142134 container cleanup 4d81e907e6037aa8a65757a7505dfaade574d84f40fcbbfde5f0eba817b5c734 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, version=8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream) 2024-09-13T22:19:22.076 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:21 smithi114 bash[113444]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0 2024-09-13T22:19:22.425 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:22 smithi114 podman[113460]: 2024-09-13 22:19:22.101315662 +0000 UTC m=+0.710508985 container remove 4d81e907e6037aa8a65757a7505dfaade574d84f40fcbbfde5f0eba817b5c734 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, release=754, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public) 2024-09-13T22:19:22.733 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:22 smithi114 podman[113549]: 2024-09-13 22:19:22.327898952 +0000 UTC m=+0.015862427 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:22.733 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:22 smithi114 podman[113549]: 2024-09-13 22:19:22.472067948 +0000 UTC m=+0.160031457 container create eaaafabe25bf327c359be48f7d6b048a7a5590d0cba6385e2e279ed0e30f7c3f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T22:19:22.733 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:22 smithi114 podman[113549]: 2024-09-13 22:19:22.658032266 +0000 UTC m=+0.345995733 container init eaaafabe25bf327c359be48f7d6b048a7a5590d0cba6385e2e279ed0e30f7c3f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T22:19:22.733 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:22 smithi114 podman[113549]: 2024-09-13 22:19:22.662661187 +0000 UTC m=+0.350624657 container start eaaafabe25bf327c359be48f7d6b048a7a5590d0cba6385e2e279ed0e30f7c3f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_BRANCH=HEAD) 2024-09-13T22:19:22.733 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:22 smithi114 podman[113549]: 2024-09-13 22:19:22.732832213 +0000 UTC m=+0.420795681 container attach eaaafabe25bf327c359be48f7d6b048a7a5590d0cba6385e2e279ed0e30f7c3f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:19:23.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:22 smithi114 ceph-mon[105598]: pgmap v106: 161 pgs: 48 peering, 113 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:19:23.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:22 smithi114 ceph-mon[105598]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T22:19:23.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:22 smithi114 ceph-mon[105598]: Health check failed: Reduced data availability: 8 pgs inactive, 47 pgs peering (PG_AVAILABILITY) 2024-09-13T22:19:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:22 smithi114 ceph-mon[108287]: pgmap v106: 161 pgs: 48 peering, 113 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:19:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:22 smithi114 ceph-mon[108287]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T22:19:23.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:22 smithi114 ceph-mon[108287]: Health check failed: Reduced data availability: 8 pgs inactive, 47 pgs peering (PG_AVAILABILITY) 2024-09-13T22:19:23.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:22 smithi114 podman[113549]: 2024-09-13 22:19:22.866675191 +0000 UTC m=+0.554638661 container died eaaafabe25bf327c359be48f7d6b048a7a5590d0cba6385e2e279ed0e30f7c3f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T22:19:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:22 smithi121 ceph-mon[83513]: pgmap v106: 161 pgs: 48 peering, 113 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-13T22:19:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:22 smithi121 ceph-mon[83513]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T22:19:23.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:22 smithi121 ceph-mon[83513]: Health check failed: Reduced data availability: 8 pgs inactive, 47 pgs peering (PG_AVAILABILITY) 2024-09-13T22:19:25.020 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:24 smithi114 podman[113572]: 2024-09-13 22:19:24.735937128 +0000 UTC m=+1.858775691 container remove eaaafabe25bf327c359be48f7d6b048a7a5590d0cba6385e2e279ed0e30f7c3f (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:19:25.020 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:24 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T22:19:25.020 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:24 smithi114 systemd[1]: Stopped Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:19:25.020 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:24 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.0.service: Consumed 13.532s CPU time. 2024-09-13T22:19:25.020 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:24 smithi114 systemd[1]: Starting Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:19:25.295 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:25 smithi114 ceph-mon[105598]: pgmap v108: 161 pgs: 10 stale+active+clean, 48 peering, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 405 B/s rd, 0 op/s 2024-09-13T22:19:25.295 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:25 smithi114 ceph-mon[108287]: pgmap v108: 161 pgs: 10 stale+active+clean, 48 peering, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 405 B/s rd, 0 op/s 2024-09-13T22:19:25.295 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 podman[113721]: 2024-09-13 22:19:25.079490303 +0000 UTC m=+0.016491525 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:25.295 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 podman[113721]: 2024-09-13 22:19:25.178536286 +0000 UTC m=+0.115537545 container create 9f4751ee4ba70ad07c786b9a1d3a721d6cf0dcdffb798d49dd948fc78f34e4f6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-09-13T22:19:25.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:25 smithi121 ceph-mon[83513]: pgmap v108: 161 pgs: 10 stale+active+clean, 48 peering, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 405 B/s rd, 0 op/s 2024-09-13T22:19:25.574 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 podman[113721]: 2024-09-13 22:19:25.439797405 +0000 UTC m=+0.376798622 container init 9f4751ee4ba70ad07c786b9a1d3a721d6cf0dcdffb798d49dd948fc78f34e4f6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T22:19:25.574 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 podman[113721]: 2024-09-13 22:19:25.443332895 +0000 UTC m=+0.380334123 container start 9f4751ee4ba70ad07c786b9a1d3a721d6cf0dcdffb798d49dd948fc78f34e4f6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:19:25.574 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 podman[113721]: 2024-09-13 22:19:25.514707915 +0000 UTC m=+0.451709159 container attach 9f4751ee4ba70ad07c786b9a1d3a721d6cf0dcdffb798d49dd948fc78f34e4f6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T22:19:26.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:26.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 bash[113721]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:26.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:26.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:25 smithi114 bash[113721]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:26.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:26 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:26.476 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:26 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:26.477 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:26 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:26.477 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:26 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:26.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:26.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:27.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:27.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:26 smithi114 bash[113721]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:27.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:26 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:27.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:26 smithi114 bash[113721]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:27.560 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-mon[108287]: pgmap v109: 161 pgs: 10 stale+active+clean, 48 peering, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:19:27.560 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:27.561 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-mon[105598]: pgmap v109: 161 pgs: 10 stale+active+clean, 48 peering, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:19:27.561 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:27.561 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:27.561 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 bash[113721]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:27.561 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:19:27.561 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 bash[113721]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:19:27.561 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T22:19:27.562 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 bash[113721]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T22:19:27.562 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:27.562 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 bash[113721]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:27.562 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[113736]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T22:19:27.562 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 bash[113721]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T22:19:27.562 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 podman[113721]: 2024-09-13 22:19:27.272447456 +0000 UTC m=+2.209448677 container died 9f4751ee4ba70ad07c786b9a1d3a721d6cf0dcdffb798d49dd948fc78f34e4f6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T22:19:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:27 smithi121 ceph-mon[83513]: pgmap v109: 161 pgs: 10 stale+active+clean, 48 peering, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:19:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:27 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:28.195 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:27 smithi114 podman[113883]: 2024-09-13 22:19:27.92727693 +0000 UTC m=+0.648238946 container remove 9f4751ee4ba70ad07c786b9a1d3a721d6cf0dcdffb798d49dd948fc78f34e4f6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T22:19:28.195 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:28 smithi114 podman[113940]: 2024-09-13 22:19:28.034339702 +0000 UTC m=+0.015942582 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:28.195 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:28 smithi114 podman[113940]: 2024-09-13 22:19:28.194870132 +0000 UTC m=+0.176472998 container create c1f1492f5fde328b73615979ef665d836ed12faf81bac623d22cbffbdb0aba9a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2) 2024-09-13T22:19:28.456 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:28 smithi114 ceph-mon[108287]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:28.456 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:28 smithi114 ceph-mon[108287]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 47 pgs peering) 2024-09-13T22:19:28.456 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:28 smithi114 ceph-mon[105598]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:28.456 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:28 smithi114 ceph-mon[105598]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 47 pgs peering) 2024-09-13T22:19:28.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:28 smithi121 ceph-mon[83513]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:28.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:28 smithi121 ceph-mon[83513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 47 pgs peering) 2024-09-13T22:19:28.776 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:28 smithi114 podman[113940]: 2024-09-13 22:19:28.455685417 +0000 UTC m=+0.437288294 container init c1f1492f5fde328b73615979ef665d836ed12faf81bac623d22cbffbdb0aba9a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:19:28.776 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:28 smithi114 podman[113940]: 2024-09-13 22:19:28.458901929 +0000 UTC m=+0.440504806 container start c1f1492f5fde328b73615979ef665d836ed12faf81bac623d22cbffbdb0aba9a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T22:19:28.776 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:28 smithi114 ceph-osd[113959]: -- 172.21.15.114:0/1691888153 <== mon.1 v2:172.21.15.114:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5608ca6fda40 con 0x5608ca8b4400 2024-09-13T22:19:28.777 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:28 smithi114 bash[113940]: c1f1492f5fde328b73615979ef665d836ed12faf81bac623d22cbffbdb0aba9a 2024-09-13T22:19:28.777 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:28 smithi114 systemd[1]: Started Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:19:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: pgmap v110: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:29.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: pgmap v110: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:29.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:29.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:29.326 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:29 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: pgmap v110: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:29.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:29.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:29.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:29.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:29 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[105598]: pgmap v111: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 529 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[105598]: pgmap v112: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 303 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:30.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:30.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:30.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[108287]: pgmap v111: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 529 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:30.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[108287]: pgmap v112: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 303 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:30.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:30.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:30.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:30 smithi121 ceph-mon[83513]: pgmap v111: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 529 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:30 smithi121 ceph-mon[83513]: pgmap v112: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 303 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:30 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:30.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:30 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:30.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:30 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:30.649 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:30 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[113955]: 2024-09-13T22:19:30.347+0000 7fa91f230640 -1 Falling back to public interface 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:19:31] "GET /metrics HTTP/1.1" 200 38169 "" "Prometheus/2.43.0" 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:19:31.731+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:32.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:32.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:32.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:32.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:32.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:32.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:32.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: pgmap v113: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:32.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: pgmap v113: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:32.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:32.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:32.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:32.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:32 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:32.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: pgmap v113: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:32.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:32.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:32.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:32.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:32.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:32.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:32.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:32 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.325 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:33.326 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.327 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.328 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:33.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:33.330 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.331 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[105598]: pgmap v114: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[105598]: pgmap v115: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:33.332 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.333 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.334 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[108287]: pgmap v114: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:33.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[108287]: pgmap v115: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:33.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:33.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:33.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:33.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:33 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:33.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:33 smithi121 ceph-mon[83513]: pgmap v114: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:33.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:33 smithi121 ceph-mon[83513]: pgmap v115: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:33.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:33 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:33.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:33 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:33.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:33 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:33.729 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:19:33.553+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.730 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.731 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:19:33.732 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:33.733 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:33.734 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:34.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[113955]: 2024-09-13T22:19:33.882+0000 7fa91f230640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-13T22:19:34.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:33 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[113955]: 2024-09-13T22:19:33.911+0000 7fa91f230640 -1 osd.0 113 log_to_monitors true 2024-09-13T22:19:34.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:34.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[108287]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:34.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:34.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:34.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:34.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:34.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:19:34.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-mon[105598]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:19:34.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:19:34.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:34 smithi121 ceph-mon[83513]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:19:34.960 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[113955]: 2024-09-13T22:19:34.638+0000 7fa916fdb640 -1 osd.0 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:19:35.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:35.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:35.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:35.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:35.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:35.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:35.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:35.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:35.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:19:35.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:35.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.582 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: pgmap v116: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:35.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: pgmap v117: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:35.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:35.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:35.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:35.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:19:35.583 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T22:19:35.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: from='osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:19:35.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:19:35.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:35.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:35.585 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:35.586 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: pgmap v116: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: pgmap v117: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: from='osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:19:35.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:35 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:19:35.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:35.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: pgmap v116: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: pgmap v117: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: from='osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:19:35.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:35 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:36.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[105598]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:19:36.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[105598]: osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335] boot 2024-09-13T22:19:36.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[105598]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T22:19:36.576 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:19:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[108287]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:19:36.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[108287]: osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335] boot 2024-09-13T22:19:36.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[108287]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T22:19:36.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:36 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:19:36.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:36 smithi121 ceph-mon[83513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:19:36.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:36 smithi121 ceph-mon[83513]: osd.0 [v2:172.21.15.114:6802/188085335,v1:172.21.15.114:6803/188085335] boot 2024-09-13T22:19:36.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:36 smithi121 ceph-mon[83513]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T22:19:36.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:36 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:19:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:37 smithi114 ceph-mon[105598]: pgmap v119: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:37 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:37.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:37 smithi114 ceph-mon[105598]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T22:19:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:37 smithi114 ceph-mon[108287]: pgmap v119: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:37 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:37.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:37 smithi114 ceph-mon[108287]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T22:19:37.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:37 smithi121 ceph-mon[83513]: pgmap v119: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:37.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:37 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:37.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:37 smithi121 ceph-mon[83513]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T22:19:38.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:38 smithi121 ceph-mon[83513]: pgmap v122: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:38.624 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:38 smithi121 ceph-mon[83513]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:38.674 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-mon[105598]: pgmap v122: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:38.674 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-mon[105598]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:38.674 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-mon[108287]: pgmap v122: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:38.674 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-mon[108287]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:39.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:19:38.790+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:39.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:39.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:39.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:39.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:39.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:19:39.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:38 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:39.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: pgmap v123: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 977 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: pgmap v124: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:39.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:39.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:39.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:39.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: pgmap v125: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: pgmap v126: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:39 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.693 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.694 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.695 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.696 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:39.697 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:39.698 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.699 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.700 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:39.701 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:39.702 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:39.703 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: pgmap v123: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 977 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: pgmap v124: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: pgmap v125: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: pgmap v126: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.704 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: pgmap v123: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 977 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: pgmap v124: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:39.705 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: pgmap v125: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: pgmap v126: 161 pgs: 37 peering, 14 active+undersized, 7 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:19:39.706 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:39 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.253 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.253 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.253 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:40 smithi121 ceph-mon[83513]: pgmap v127: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:19:41.253 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:40 smithi121 ceph-mon[83513]: pgmap v128: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:19:41.253 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.253 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.253 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:40 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:41.289 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.289 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.289 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[105598]: pgmap v127: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[105598]: pgmap v128: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:40 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:41 smithi114 ceph-mon[108287]: pgmap v127: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:41 smithi114 ceph-mon[108287]: pgmap v128: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:41 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:41 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:41.290 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:41 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:41.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:41 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:19:41] "GET /metrics HTTP/1.1" 200 38259 "" "Prometheus/2.43.0" 2024-09-13T22:19:42.309 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:41 smithi114 ceph-mon[105598]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T22:19:42.310 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:42 smithi114 ceph-mon[108287]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T22:19:42.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:41 smithi121 ceph-mon[83513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T22:19:43.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:43 smithi114 ceph-mon[108287]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:43.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:43 smithi114 ceph-mon[105598]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:43.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:43 smithi121 ceph-mon[83513]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:44.324 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:19:44.178+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:44.325 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:44.326 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:44.327 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:19:44.328 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:44.329 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:44.330 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.261 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-09-13T22:19:45.261 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:45.262 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:45.263 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:45.264 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:45.265 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:45.266 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:45.267 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:45.268 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[105598]: Upgrade: osd.0 is safe to restart 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:45.269 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:45.270 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:45.271 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:45.272 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:45 smithi114 ceph-mon[108287]: Upgrade: osd.0 is safe to restart 2024-09-13T22:19:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-09-13T22:19:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:45.371 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:45.372 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:45.373 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:45.374 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:45.375 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:19:45.376 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:45 smithi121 ceph-mon[83513]: Upgrade: osd.0 is safe to restart 2024-09-13T22:19:46.710 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:19:47.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[105598]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 727 B/s rd, 0 op/s 2024-09-13T22:19:47.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[108287]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 727 B/s rd, 0 op/s 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:47 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:47 smithi121 ceph-mon[83513]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 727 B/s rd, 0 op/s 2024-09-13T22:19:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:47 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:47 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:47.458 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:19:47.458 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 0s ago 32m - - 2024-09-13T22:19:47.458 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 8s ago 32m - - 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (18m) 0s ago 26m 25.7M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (4m) 8s ago 26m 80.8M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 0s ago 25m 63.0M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (5m) 8s ago 31m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (15m) 0s ago 33m 510M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (2m) 0s ago 33m 51.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 96aac05a92f3 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (3m) 8s ago 31m 42.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (99s) 0s ago 32m 37.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 6670a4db3663 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (19m) 0s ago 27m 14.8M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (18m) 8s ago 27m 15.2M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (19s) 0s ago 31m 41.7M 4096M 19.1.1-324-gecab7be5 0652ff2b5d78 c1f1492f5fde 2024-09-13T22:19:47.459 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (30m) 0s ago 30m 58.4M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (30m) 0s ago 30m 51.0M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (30m) 0s ago 30m 53.5M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (29m) 8s ago 29m 55.7M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (29m) 8s ago 29m 56.2M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (29m) 8s ago 29m 53.3M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (28m) 8s ago 28m 54.1M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (5m) 8s ago 27m 67.2M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (26m) 0s ago 26m 77.6M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:19:47.460 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (26m) 8s ago 26m 78.8M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:19:47.866 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:19:47.866 INFO:teuthology.orchestra.run.smithi114.stdout: "mon": { 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: "mgr": { 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: "osd": { 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:47.867 INFO:teuthology.orchestra.run.smithi114.stdout: "rgw": { 2024-09-13T22:19:47.868 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:19:47.868 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2024-09-13T22:19:47.868 INFO:teuthology.orchestra.run.smithi114.stdout: "overall": { 2024-09-13T22:19:47.868 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T22:19:47.868 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T22:19:47.868 INFO:teuthology.orchestra.run.smithi114.stdout: } 2024-09-13T22:19:47.868 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:19:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:19:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='client.74185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: Upgrade: Updating osd.0 2024-09-13T22:19:48.074 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: Deploying daemon osd.0 on smithi114 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='client.64221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/2497520736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='client.74185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:48.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: Upgrade: Updating osd.0 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: Deploying daemon osd.0 on smithi114 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='client.64221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:48 smithi114 ceph-mon[108287]: from='client.? 172.21.15.114:0/2497520736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout: "in_progress": true, 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout: "services_complete": [], 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout: "progress": "1/8 daemons upgraded", 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout: "is_paused": false 2024-09-13T22:19:48.220 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-09-13T22:19:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:19:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:19:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='client.74185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: Upgrade: Updating osd.0 2024-09-13T22:19:48.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:19:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: Deploying daemon osd.0 on smithi114 2024-09-13T22:19:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='client.64221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:48.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:48 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/2497520736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:19:49.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[105598]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:19:49.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[105598]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:19:49.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[105598]: from='client.64227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:49.298 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[105598]: from='client.74209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:49.298 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[108287]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:19:49.298 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[108287]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:19:49.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[108287]: from='client.64227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:49.299 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:49 smithi114 ceph-mon[108287]: from='client.74209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:49 smithi121 ceph-mon[83513]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-13T22:19:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:49 smithi121 ceph-mon[83513]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:19:49.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:49 smithi121 ceph-mon[83513]: from='client.64227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:49.372 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:49 smithi121 ceph-mon[83513]: from='client.74209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:19:50.824 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:50 smithi114 systemd[1]: Stopping Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:19:50.825 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[113955]: 2024-09-13T22:19:50.698+0000 7fa91c1c8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:19:50.825 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[113955]: 2024-09-13T22:19:50.698+0000 7fa91c1c8640 -1 osd.0 118 *** Got signal Terminated *** 2024-09-13T22:19:50.825 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:50 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[113955]: 2024-09-13T22:19:50.698+0000 7fa91c1c8640 -1 osd.0 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:19:51.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:51 smithi114 ceph-mon[108287]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:19:51.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:51 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:51.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:51 smithi114 ceph-mon[108287]: osd.0 marked itself down and dead 2024-09-13T22:19:51.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:51 smithi114 ceph-mon[105598]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:19:51.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:51 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:51.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:51 smithi114 ceph-mon[105598]: osd.0 marked itself down and dead 2024-09-13T22:19:51.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:51 smithi121 ceph-mon[83513]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-09-13T22:19:51.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:51 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:51.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:51 smithi121 ceph-mon[83513]: osd.0 marked itself down and dead 2024-09-13T22:19:51.942 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:19:51] "GET /metrics HTTP/1.1" 200 38259 "" "Prometheus/2.43.0" 2024-09-13T22:19:51.942 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:51 smithi114 podman[116824]: 2024-09-13 22:19:51.66891625 +0000 UTC m=+1.050395297 container died c1f1492f5fde328b73615979ef665d836ed12faf81bac623d22cbffbdb0aba9a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T22:19:52.245 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:52 smithi114 podman[116824]: 2024-09-13 22:19:52.106437648 +0000 UTC m=+1.487916694 container cleanup c1f1492f5fde328b73615979ef665d836ed12faf81bac623d22cbffbdb0aba9a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, RELEASE=squid-ecab7be, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T22:19:52.245 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:52 smithi114 bash[116824]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0 2024-09-13T22:19:52.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:52 smithi114 ceph-mon[105598]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:19:52.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:52 smithi114 ceph-mon[105598]: osdmap e119: 8 total, 7 up, 8 in 2024-09-13T22:19:52.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:52 smithi114 ceph-mon[108287]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:19:52.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:52 smithi114 ceph-mon[108287]: osdmap e119: 8 total, 7 up, 8 in 2024-09-13T22:19:52.575 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:52 smithi114 podman[116839]: 2024-09-13 22:19:52.307790297 +0000 UTC m=+0.633014485 container remove c1f1492f5fde328b73615979ef665d836ed12faf81bac623d22cbffbdb0aba9a (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True) 2024-09-13T22:19:52.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:52 smithi121 ceph-mon[83513]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:19:52.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:52 smithi121 ceph-mon[83513]: osdmap e119: 8 total, 7 up, 8 in 2024-09-13T22:19:53.045 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:52 smithi114 podman[116929]: 2024-09-13 22:19:52.620254624 +0000 UTC m=+0.015783477 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:53.045 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:52 smithi114 podman[116929]: 2024-09-13 22:19:52.783793957 +0000 UTC m=+0.179322854 container create cac66e3e8e61a434861c69b279b24ac264833728c56b40bd9e691239d97b7992 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T22:19:53.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:53 smithi114 ceph-mon[105598]: pgmap v138: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-13T22:19:53.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:53 smithi114 ceph-mon[105598]: osdmap e120: 8 total, 7 up, 8 in 2024-09-13T22:19:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:53 smithi114 ceph-mon[108287]: pgmap v138: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-13T22:19:53.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:53 smithi114 ceph-mon[108287]: osdmap e120: 8 total, 7 up, 8 in 2024-09-13T22:19:53.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:53 smithi114 podman[116929]: 2024-09-13 22:19:53.04438376 +0000 UTC m=+0.439912613 container init cac66e3e8e61a434861c69b279b24ac264833728c56b40bd9e691239d97b7992 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:19:53.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:53 smithi114 podman[116929]: 2024-09-13 22:19:53.048381595 +0000 UTC m=+0.443910454 container start cac66e3e8e61a434861c69b279b24ac264833728c56b40bd9e691239d97b7992 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T22:19:53.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:53 smithi114 podman[116929]: 2024-09-13 22:19:53.144451362 +0000 UTC m=+0.539980218 container attach cac66e3e8e61a434861c69b279b24ac264833728c56b40bd9e691239d97b7992 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2) 2024-09-13T22:19:53.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:53 smithi114 podman[116929]: 2024-09-13 22:19:53.238119921 +0000 UTC m=+0.633648786 container died cac66e3e8e61a434861c69b279b24ac264833728c56b40bd9e691239d97b7992 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T22:19:53.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:53 smithi121 ceph-mon[83513]: pgmap v138: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-13T22:19:53.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:53 smithi121 ceph-mon[83513]: osdmap e120: 8 total, 7 up, 8 in 2024-09-13T22:19:54.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 podman[116952]: 2024-09-13 22:19:54.065284649 +0000 UTC m=+0.821168387 container remove cac66e3e8e61a434861c69b279b24ac264833728c56b40bd9e691239d97b7992 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:19:54.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T22:19:54.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 systemd[1]: Stopped Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:19:54.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.0.service: Consumed 1.724s CPU time. 2024-09-13T22:19:54.325 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 systemd[1]: Starting Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:19:54.623 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 podman[117070]: 2024-09-13 22:19:54.399172589 +0000 UTC m=+0.015980198 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:54.623 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 podman[117070]: 2024-09-13 22:19:54.524984017 +0000 UTC m=+0.141791620 container create c9090367ef5b78dfd2f0f7ed1dca9104bad6ffb135d3d70e38301cdae2d47de9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T22:19:54.877 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 podman[117070]: 2024-09-13 22:19:54.794240851 +0000 UTC m=+0.411048460 container init c9090367ef5b78dfd2f0f7ed1dca9104bad6ffb135d3d70e38301cdae2d47de9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:19:54.878 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 podman[117070]: 2024-09-13 22:19:54.798690886 +0000 UTC m=+0.415498494 container start c9090367ef5b78dfd2f0f7ed1dca9104bad6ffb135d3d70e38301cdae2d47de9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-13T22:19:54.878 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 podman[117070]: 2024-09-13 22:19:54.877365133 +0000 UTC m=+0.494172737 container attach c9090367ef5b78dfd2f0f7ed1dca9104bad6ffb135d3d70e38301cdae2d47de9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T22:19:55.258 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:55.258 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 bash[117070]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:55.258 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:55.258 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:54 smithi114 bash[117070]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:19:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:55 smithi114 ceph-mon[105598]: pgmap v140: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:19:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:55 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:55.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:55 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:55.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:55 smithi114 ceph-mon[108287]: pgmap v140: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:19:55.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:55 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:55.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:55 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:55 smithi121 ceph-mon[83513]: pgmap v140: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-13T22:19:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:55 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:55.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:55 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:19:56.574 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:56.574 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 bash[117070]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:56.574 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:56.574 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 bash[117070]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:56.939 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:56.939 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 bash[117070]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:19:56.939 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 bash[117070]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 bash[117070]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 bash[117070]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate[117086]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 bash[117070]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T22:19:56.940 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:56 smithi114 podman[117070]: 2024-09-13 22:19:56.644742048 +0000 UTC m=+2.261549656 container died c9090367ef5b78dfd2f0f7ed1dca9104bad6ffb135d3d70e38301cdae2d47de9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be) 2024-09-13T22:19:57.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:57 smithi114 ceph-mon[105598]: pgmap v141: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T22:19:57.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:57 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:57.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:57 smithi114 ceph-mon[108287]: pgmap v141: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T22:19:57.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:57 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:57.575 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 podman[117242]: 2024-09-13 22:19:57.256430716 +0000 UTC m=+0.602335296 container remove c9090367ef5b78dfd2f0f7ed1dca9104bad6ffb135d3d70e38301cdae2d47de9 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T22:19:57.575 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 podman[117302]: 2024-09-13 22:19:57.363066355 +0000 UTC m=+0.015899977 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:19:57.575 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 podman[117302]: 2024-09-13 22:19:57.474063363 +0000 UTC m=+0.126897025 container create 4efb123ca48e990e07823830cd6ade7673ba10ba5251f064b27e2b8ad4efbc62 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:19:57.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:57 smithi121 ceph-mon[83513]: pgmap v141: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-13T22:19:57.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:57 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:19:58.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 podman[117302]: 2024-09-13 22:19:57.701112014 +0000 UTC m=+0.353945634 container init 4efb123ca48e990e07823830cd6ade7673ba10ba5251f064b27e2b8ad4efbc62 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:19:58.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 podman[117302]: 2024-09-13 22:19:57.7043278 +0000 UTC m=+0.357161416 container start 4efb123ca48e990e07823830cd6ade7673ba10ba5251f064b27e2b8ad4efbc62 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True) 2024-09-13T22:19:58.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 ceph-osd[117321]: -- 172.21.15.114:0/4211641546 <== mon.0 v2:172.21.15.114:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556b133f1a40 con 0x556b141b8000 2024-09-13T22:19:58.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 bash[117302]: 4efb123ca48e990e07823830cd6ade7673ba10ba5251f064b27e2b8ad4efbc62 2024-09-13T22:19:58.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:57 smithi114 systemd[1]: Started Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: pgmap v142: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: pgmap v143: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 773 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: pgmap v144: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.075 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[105598]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: pgmap v142: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.076 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: pgmap v143: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 773 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: pgmap v144: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:59.077 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:19:58 smithi114 ceph-mon[108287]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:59.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: pgmap v142: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.106 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: pgmap v143: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 773 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: pgmap v144: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:19:59.107 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:19:58 smithi121 ceph-mon[83513]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:19:59.824 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[117317]: 2024-09-13T22:19:59.587+0000 7f25aca71640 -1 Falling back to public interface 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:19:59.531+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:59.825 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:59.826 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:19:59.827 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:19:59.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:19:59.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:19:59.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:19:59.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:19:59.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:19:59.828 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:19:59.829 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:19:59.830 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:19:59 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:00.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pgmap v145: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 747 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:00.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pgmap v146: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:00.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: osd.0 (root=default,host=smithi114) is down 2024-09-13T22:20:00.326 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: [WRN] PG_DEGRADED: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 2.5 is active+undersized+degraded, acting [2,4] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-13T22:20:00.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pgmap v145: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 747 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pgmap v146: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:00.328 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: osd.0 (root=default,host=smithi114) is down 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: [WRN] PG_DEGRADED: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 2.5 is active+undersized+degraded, acting [2,4] 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-13T22:20:00.329 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-13T22:20:00.330 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[105598]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-13T22:20:00.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:00 smithi114 ceph-mon[108287]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-09-13T22:20:00.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.353 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pgmap v145: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 747 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:00.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pgmap v146: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:00.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:00.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:00.354 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:00.355 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:00.355 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:00.355 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:00.355 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:00.355 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded 2024-09-13T22:20:00.355 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: daemon iscsi.foo.smithi114.lemsof on smithi114 is in unknown state 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: osd.0 (root=default,host=smithi114) is down 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: [WRN] PG_DEGRADED: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-13T22:20:00.356 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 2.5 is active+undersized+degraded, acting [2,4] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-13T22:20:00.357 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-13T22:20:00.358 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:00 smithi121 ceph-mon[83513]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-09-13T22:20:01.324 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:01.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:01.324 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:01.325 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:01.326 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:01.327 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:01.328 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:20:01.329 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:01.330 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.331 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[108287]: pgmap v147: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:01.332 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:01.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:01.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:01.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:01.333 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[105598]: pgmap v147: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:01.334 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[108287]: pgmap v148: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:01.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:01.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:01.335 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:01.335 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[105598]: pgmap v148: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:01.335 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:01.335 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:01.335 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:01.339 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:01 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:01.339 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:01.339 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:01.340 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:20:01.341 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:01.342 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:01.343 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:01 smithi121 ceph-mon[83513]: pgmap v147: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:01 smithi121 ceph-mon[83513]: pgmap v148: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:01 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:01 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:01.344 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:01 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:01.972 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:01 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:20:01] "GET /metrics HTTP/1.1" 200 38254 "" "Prometheus/2.43.0" 2024-09-13T22:20:02.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:02.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:02.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:02.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:02.254 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:20:02.030+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:02.255 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:02.256 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:02.257 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:02.257 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:02.257 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:02.257 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:02.257 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:02.257 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:20:02.258 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:02.259 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:02.260 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:02.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:02 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:02.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:02 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:02.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:02 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:02.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:02 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:02.371 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:02 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:03.167 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:02 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[117317]: 2024-09-13T22:20:02.909+0000 7f25aca71640 -1 osd.0 118 log_to_monitors true 2024-09-13T22:20:03.167 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[117317]: 2024-09-13T22:20:03.134+0000 7f25a481c640 -1 osd.0 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: pgmap v149: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.168 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.169 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:03.170 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:03.171 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.172 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: pgmap v149: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:03.173 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:03.174 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:03.175 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:03.176 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.177 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: pgmap v150: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: pgmap v151: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[105598]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: pgmap v150: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: pgmap v151: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:03.178 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:03.179 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:03.179 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:03.179 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:03 smithi114 ceph-mon[108287]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:20:03.324 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: pgmap v149: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.324 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:03.324 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:03.324 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:03.324 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:03.325 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:20:03.326 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:03.328 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:03.328 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:03.328 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:20:03.328 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:03.328 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:20:03.328 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.329 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:03.330 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: pgmap v150: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: pgmap v151: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:03.331 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:03 smithi121 ceph-mon[83513]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:20:04.221 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:04 smithi114 ceph-mon[108287]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:20:04.221 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:04 smithi114 ceph-mon[108287]: osdmap e121: 8 total, 7 up, 8 in 2024-09-13T22:20:04.221 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:04 smithi114 ceph-mon[108287]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:20:04.221 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:04 smithi114 ceph-mon[105598]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:20:04.221 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:04 smithi114 ceph-mon[105598]: osdmap e121: 8 total, 7 up, 8 in 2024-09-13T22:20:04.221 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:04 smithi114 ceph-mon[105598]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:20:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:04 smithi121 ceph-mon[83513]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:20:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:04 smithi121 ceph-mon[83513]: osdmap e121: 8 total, 7 up, 8 in 2024-09-13T22:20:04.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:04 smithi121 ceph-mon[83513]: from='osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi114", "root=default"]}]: dispatch 2024-09-13T22:20:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[105598]: pgmap v153: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[105598]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:20:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[105598]: osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185] boot 2024-09-13T22:20:05.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[105598]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T22:20:05.325 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:20:05.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[108287]: pgmap v153: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:05.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[108287]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:20:05.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[108287]: osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185] boot 2024-09-13T22:20:05.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[108287]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T22:20:05.325 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:20:05.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:05 smithi121 ceph-mon[83513]: pgmap v153: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 68/627 objects degraded (10.845%) 2024-09-13T22:20:05.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:05 smithi121 ceph-mon[83513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:20:05.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:05 smithi121 ceph-mon[83513]: osd.0 [v2:172.21.15.114:6802/1229000185,v1:172.21.15.114:6803/1229000185] boot 2024-09-13T22:20:05.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:05 smithi121 ceph-mon[83513]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T22:20:05.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:05 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:20:06.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:20:05.795+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:06.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:06.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:06.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:06.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:06.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:06.075 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:06.076 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:06.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:06.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:06.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:06.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:06.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:06.077 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:06.078 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:06.079 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:06.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:06.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:06.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:06.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:06.080 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:06.434 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T22:20:06.434 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:06.434 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:06.434 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:06.434 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:06.434 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:06.435 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:06 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T22:20:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:06.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:06.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:06.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:06 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.441 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.442 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:07.443 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:07.444 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:07.445 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:07.446 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:07.447 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[105598]: pgmap v156: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:07.448 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:07.449 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[108287]: pgmap v156: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[105598]: pgmap v157: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:07.450 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[105598]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[108287]: pgmap v157: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[108287]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:07.451 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:07 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:07.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:07 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:07.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:07.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:07.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:07.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:07.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:07 smithi121 ceph-mon[83513]: pgmap v156: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:07 smithi121 ceph-mon[83513]: pgmap v157: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:07 smithi121 ceph-mon[83513]: Metadata not up to date on all hosts. Skipping non agent specs 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:07 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:07 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:07.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:07 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[105598]: pgmap v158: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[105598]: pgmap v159: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[108287]: pgmap v158: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:20:08.653 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[108287]: pgmap v159: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:20:08.654 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:08 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:08 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:08 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:08.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:08 smithi121 ceph-mon[83513]: pgmap v158: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:20:08.871 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:08 smithi121 ceph-mon[83513]: pgmap v159: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-09-13T22:20:08.872 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:08 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:09.772 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:09 smithi121 ceph-mon[83513]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T22:20:09.825 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:09 smithi114 ceph-mon[105598]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T22:20:09.825 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:09 smithi114 ceph-mon[108287]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T22:20:11.518 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[105598]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-09-13T22:20:11.518 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:11.518 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:20:11.518 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[105598]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T22:20:11.518 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[108287]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-09-13T22:20:11.518 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:11.518 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:20:11.519 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-mon[108287]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T22:20:11.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:11 smithi121 ceph-mon[83513]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-09-13T22:20:11.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:11 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:11.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:11 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:20:11.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:11 smithi121 ceph-mon[83513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-09-13T22:20:11.824 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:11 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:20:11] "GET /metrics HTTP/1.1" 200 38256 "" "Prometheus/2.43.0" 2024-09-13T22:20:13.486 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-mon[105598]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:20:13.486 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-mon[108287]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:20:13.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:20:13.290+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:13.486 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:13.487 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:13.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:13.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:13.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:13.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:13.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:13.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:13.488 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:20:13.489 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:13.490 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:13.491 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:13 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:13.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:13 smithi121 ceph-mon[83513]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:20:14.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:14.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:14.574 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:14.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:14.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:14.577 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:14.578 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:14.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.580 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.581 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:14.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:14.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:20:14.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:20:14.585 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:20:14.586 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:20:14.587 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.588 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:20:14.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:20:14.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:14.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:14.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:14.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:14.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.591 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.592 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:14 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:14.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:14.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:14.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:14.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:20:14.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:20:14.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:20:14.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:14.629 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:14.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:14.630 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:14 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: Upgrade: Finalizing container_image settings 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: Upgrade: Complete! 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:20:15.384 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:15.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:20:15.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: Upgrade: Finalizing container_image settings 2024-09-13T22:20:15.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: Upgrade: Complete! 2024-09-13T22:20:15.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-09-13T22:20:15.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-09-13T22:20:15.385 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:15.386 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:15.386 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:20:15.386 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:20:15.386 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:15 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:20:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: Upgrade: Finalizing container_image settings 2024-09-13T22:20:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: Upgrade: Complete! 2024-09-13T22:20:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-09-13T22:20:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-09-13T22:20:15.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-13T22:20:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:20:15.622 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:15 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:20:16.380+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:16.575 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:16.576 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:16.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:16.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:16.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:16.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:16.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:20:16.577 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:20:16.578 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:16.579 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:16.580 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:16 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:17.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[105598]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[108287]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:17.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:17.576 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:17 smithi121 ceph-mon[83513]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:20:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:17 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:20:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi114.lemsof", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-13T22:20:17.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:17 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:20:18.308 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: 2024-09-13T22:20:17.973+0000 7fcdfd92f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.308 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.309 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.wait_async( 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:18.310 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return future.result(timeout) 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return self.__get_result() 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise self._exception 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: out, err, code = await self._run_cephadm( 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: raise OrchestratorError( 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Creating ceph-iscsi config... 2024-09-13T22:20:18.311 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: Traceback (most recent call last): 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: exec(code, run_globals) 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.312 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.313 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.313 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.313 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.313 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.313 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:17 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.575 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.576 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:18.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:18.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:18.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:18.578 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:18.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:18.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:18.582 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:18.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:18.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:18.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:18.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:18.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:18.583 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:18.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:18.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:18.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:18.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:20:18.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:18.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:18.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:18.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:18.585 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.586 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:18.586 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.586 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.586 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.586 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.586 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.587 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.587 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.587 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.587 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.587 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:18.587 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.588 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.588 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.588 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.588 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.588 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.588 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.589 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.589 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[105598]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:20:18.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[105598]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:20:18.589 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[105598]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:18.590 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[105598]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:18.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:18.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:18.590 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:18.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:20:18.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:18.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:18.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:18.591 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:18.592 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.592 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:18.592 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.592 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.592 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.593 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.593 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.593 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.593 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.593 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.593 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.594 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:18.594 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.594 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.594 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.594 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.594 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.595 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.595 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.595 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.595 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.595 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[108287]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:20:18.595 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[108287]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:20:18.596 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[108287]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:18.596 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:18 smithi114 ceph-mon[108287]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:18.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:18 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:18.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:18.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:20:18.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:18.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:18.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:18.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:18.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:18.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:18.627 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:18.627 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:18.627 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:18.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:18 smithi121 ceph-mon[83513]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-13T22:20:18.627 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:18 smithi121 ceph-mon[83513]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-13T22:20:18.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:18 smithi121 ceph-mon[83513]: Reconfiguring iscsi.foo.smithi114.lemsof (dependencies changed)... 2024-09-13T22:20:18.628 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:18 smithi121 ceph-mon[83513]: Reconfiguring daemon iscsi.foo.smithi114.lemsof on smithi114 2024-09-13T22:20:19.371 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:20:19.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[108287]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:19.574 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:19.574 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:19.574 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:19.574 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:19.574 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:19.575 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:19.576 INFO:journalctl@ceph.mon.c.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[105598]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:19.577 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.wait_async( 2024-09-13T22:20:19.578 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.a.smithi114.stdout: return future.result(timeout) 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise self._exception 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:19.579 INFO:journalctl@ceph.mon.c.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:19.580 INFO:journalctl@ceph.mon.c.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.581 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: return self.__get_result() 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise self._exception 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: raise OrchestratorError( 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:19.582 INFO:journalctl@ceph.mon.a.smithi114.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: exec(code, run_globals) 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:19.583 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:19.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:19.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:19.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.584 INFO:journalctl@ceph.mon.a.smithi114.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.584 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:19 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:19 smithi121 ceph-mon[83513]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:19.621 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:19.622 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.wait_async( 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-13T22:20:19.623 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return future.result(timeout) 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: return self.__get_result() 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise self._exception 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: out, err, code = await self._run_cephadm( 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: raise OrchestratorError( 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi114.lemsof ... 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Creating ceph-iscsi config... 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/iscsi-gateway.cfg 2024-09-13T22:20:19.624 INFO:journalctl@ceph.mon.b.smithi121.stdout: Write file: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/iscsi.foo.smithi114.lemsof/tcmu-runner-entrypoint.sh 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Non-zero exit code 1 from systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: systemctl: stderr See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: Traceback (most recent call last): 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: return _run_code(code, main_globals, None, 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: exec(code, run_globals) 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5579, in 2024-09-13T22:20:19.625 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 5567, in main 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: File "/tmp/tmpcdw9lrs8.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: RuntimeError: Failed command: systemctl restart ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof: Job for ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service failed because the control process exited with error code. 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout: See "systemctl status ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" and "journalctl -xeu ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@iscsi.foo.smithi114.lemsof.service" for details. 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:19 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:19 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:19.626 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:19 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:20.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[105598]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-09-13T22:20:20.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[105598]: from='client.74221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:20.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[105598]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:20:20.575 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[105598]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:20:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[108287]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-09-13T22:20:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[108287]: from='client.74221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[108287]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:20:20.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:20 smithi114 ceph-mon[108287]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:20:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:20 smithi121 ceph-mon[83513]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-09-13T22:20:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:20 smithi121 ceph-mon[83513]: from='client.74221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:20 smithi121 ceph-mon[83513]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:20:20.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:20 smithi121 ceph-mon[83513]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-09-13T22:20:21.337 INFO:teuthology.orchestra.run.smithi114.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:20:21.337 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi114 smithi114 running 7s ago 32m - - 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:agent.smithi121 smithi121 running 2s ago 32m - - 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:alertmanager.a smithi114 *:9093,9094 running (19m) 7s ago 27m 25.7M - 0.25.0 c8568f914cd2 4b3ccb3d3992 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:grafana.a smithi121 *:3000 running (4m) 2s ago 27m 80.3M - 9.4.12 2bacad6d85d8 6ee3553a890c 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:iscsi.foo.smithi114.lemsof smithi114 unknown 7s ago 26m 63.0M - 3.5 e1d6a67b021e b0957dbccc51 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.x smithi121 *:8443,9283,8765 running (5m) 2s ago 32m 441M - 19.1.1-324-gecab7be5 0652ff2b5d78 9dc33ce81ccf 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:mgr.y smithi114 *:8443,9283,8765 running (15m) 7s ago 33m 511M - 19.1.1-324-gecab7be5 0652ff2b5d78 8cc35749ec26 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:mon.a smithi114 running (2m) 7s ago 34m 53.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 96aac05a92f3 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:mon.b smithi121 running (3m) 2s ago 32m 45.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 2847ae29281d 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:mon.c smithi114 running (2m) 7s ago 32m 38.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 6670a4db3663 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.a smithi114 *:9100 running (19m) 7s ago 28m 14.8M - 1.5.0 0da6a335fe13 43f14e3df9a3 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:node-exporter.b smithi121 *:9100 running (18m) 2s ago 28m 14.6M - 1.5.0 0da6a335fe13 fa1cf72b5089 2024-09-13T22:20:21.338 INFO:teuthology.orchestra.run.smithi114.stdout:osd.0 smithi114 running (23s) 7s ago 31m 43.5M 4096M 19.1.1-324-gecab7be5 0652ff2b5d78 4efb123ca48e 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:osd.1 smithi114 running (31m) 7s ago 31m 60.0M 4096M 17.2.0 e1d6a67b021e 4c38b33df90e 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:osd.2 smithi114 running (31m) 7s ago 31m 52.0M 4096M 17.2.0 e1d6a67b021e d4e942a420ea 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:osd.3 smithi114 running (30m) 7s ago 30m 53.8M 4096M 17.2.0 e1d6a67b021e 60269299bda8 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:osd.4 smithi121 running (30m) 2s ago 30m 56.7M 4096M 17.2.0 e1d6a67b021e 4e71e78263dd 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:osd.5 smithi121 running (29m) 2s ago 29m 57.1M 4096M 17.2.0 e1d6a67b021e 95e89c636120 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:osd.6 smithi121 running (29m) 2s ago 29m 54.5M 4096M 17.2.0 e1d6a67b021e 369b1a1170ed 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:osd.7 smithi121 running (29m) 2s ago 29m 55.2M 4096M 17.2.0 e1d6a67b021e c365a04abea4 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:prometheus.a smithi121 *:9095 running (6m) 2s ago 28m 67.3M - 2.43.0 a07b618ecd1d 7738fb682bbc 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi114.vrzgda smithi114 *:8000 running (26m) 7s ago 26m 77.7M - 17.2.0 e1d6a67b021e 0847004ab33b 2024-09-13T22:20:21.339 INFO:teuthology.orchestra.run.smithi114.stdout:rgw.foo.smithi121.ihcnui smithi121 *:8000 running (26m) 2s ago 26m 79.0M - 17.2.0 e1d6a67b021e 57c6fc891e6f 2024-09-13T22:20:21.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:21 smithi114 ceph-mon[105598]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:20:21.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:21 smithi114 ceph-mon[108287]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:20:21.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:21 smithi121 ceph-mon[83513]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-13T22:20:21.944 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:21 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:20:21] "GET /metrics HTTP/1.1" 200 38256 "" "Prometheus/2.43.0" 2024-09-13T22:20:21.969 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-13T22:20:22.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:22 smithi114 ceph-mon[105598]: from='client.74227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:22.575 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:22 smithi114 ceph-mon[108287]: from='client.74227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:22.591 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:22 smithi121 ceph-mon[83513]: from='client.74227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:23.370 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:23 smithi114 ceph-mon[105598]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 0 op/s 2024-09-13T22:20:23.371 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:23 smithi114 ceph-mon[108287]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 0 op/s 2024-09-13T22:20:23.553 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:23 smithi121 ceph-mon[83513]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 0 op/s 2024-09-13T22:20:23.892 INFO:teuthology.orchestra.run.smithi114.stdout:true 2024-09-13T22:20:24.539 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:24 smithi114 ceph-mon[105598]: from='client.? 172.21.15.114:0/3193022738' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:24.539 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:24 smithi114 ceph-mon[108287]: from='client.? 172.21.15.114:0/3193022738' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:24.540 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-09-13T22:20:24.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:24 smithi121 ceph-mon[83513]: from='client.? 172.21.15.114:0/3193022738' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:20:26.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:26 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:26.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:26 smithi114 ceph-mon[105598]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:20:26.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:26 smithi114 ceph-mon[105598]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-09-13T22:20:26.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:26 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:26.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:26 smithi114 ceph-mon[108287]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:20:26.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:26 smithi114 ceph-mon[108287]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-09-13T22:20:26.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' 2024-09-13T22:20:26.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:26 smithi121 ceph-mon[83513]: from='mgr.25675 172.21.15.114:0/1440120028' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:20:26.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:26 smithi121 ceph-mon[83513]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-09-13T22:20:27.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:27 smithi114 ceph-mon[105598]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:27.574 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:27 smithi114 ceph-mon[108287]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:27.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:27 smithi121 ceph-mon[83513]: from='client.25399 -' entity='client.iscsi.foo.smithi114.lemsof' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-13T22:20:27.941 INFO:teuthology.orchestra.run.smithi114.stdout:false 2024-09-13T22:20:28.487 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:28 smithi114 ceph-mon[105598]: from='client.64269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:28.488 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:28 smithi114 ceph-mon[105598]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:20:28.488 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:28 smithi114 ceph-mon[108287]: from='client.64269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:28.488 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:28 smithi114 ceph-mon[108287]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:20:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:20:28.511 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks/cephadm.py", line 1476, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi114 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c \'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e \'"\'"\'.up_to_date | length == 7\'"\'"\'\'' 2024-09-13T22:20:28.756 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=316eac3955314b3db4f1fb5429dce82f Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks/cephadm.py", line 1476, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi114 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c \'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e \'"\'"\'.up_to_date | length == 7\'"\'"\'\'' 2024-09-13T22:20:28.759 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T22:20:29.076 INFO:tasks.cephadm:Teardown begin 2024-09-13T22:20:29.077 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:20:29.093 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:28 smithi121 ceph-mon[83513]: from='client.64269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:20:29.094 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:28 smithi121 ceph-mon[83513]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-13T22:20:29.125 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:20:29.163 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T22:20:29.163 DEBUG:teuthology.orchestra.run.smithi114:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T22:20:29.183 DEBUG:teuthology.orchestra.run.smithi121:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T22:20:29.220 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T22:20:29.221 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T22:20:29.221 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a 2024-09-13T22:20:29.574 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:29 smithi114 systemd[1]: Stopping Ceph mon.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:29.930 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[105591]: 2024-09-13T22:20:29.673+0000 7f7618297640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:20:29.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a[105591]: 2024-09-13T22:20:29.673+0000 7f7618297640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T22:20:29.931 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:29 smithi114 podman[120649]: 2024-09-13 22:20:29.79653009 +0000 UTC m=+0.379550066 container died 96aac05a92f368526c3157e8db5094629b5b16ae2a3a6fba6c92f6db8be184fe (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T22:20:29.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:20:29] ENGINE Bus STOPPING 2024-09-13T22:20:29.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:20:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T22:20:29.931 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:20:29] ENGINE Bus STOPPED 2024-09-13T22:20:29.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:20:29] ENGINE Bus STARTING 2024-09-13T22:20:29.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:20:29] ENGINE Serving on http://:::9283 2024-09-13T22:20:29.932 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:29 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: [13/Sep/2024:22:20:29] ENGINE Bus STARTED 2024-09-13T22:20:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:30 smithi114 podman[120649]: 2024-09-13 22:20:30.157225332 +0000 UTC m=+0.740245339 container cleanup 96aac05a92f368526c3157e8db5094629b5b16ae2a3a6fba6c92f6db8be184fe (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be) 2024-09-13T22:20:30.324 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:30 smithi114 bash[120649]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a 2024-09-13T22:20:30.743 INFO:journalctl@ceph.mon.a.smithi114.stdout:Sep 13 22:20:30 smithi114 podman[120712]: 2024-09-13 22:20:30.433321201 +0000 UTC m=+0.631762999 container remove 96aac05a92f368526c3157e8db5094629b5b16ae2a3a6fba6c92f6db8be184fe (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:20:31.733 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T22:20:31.803 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:20:31.804 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T22:20:31.804 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T22:20:31.804 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.c 2024-09-13T22:20:32.074 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:31 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y[70604]: ::ffff:172.21.15.121 - - [13/Sep/2024:22:20:31] "GET /metrics HTTP/1.1" 200 38255 "" "Prometheus/2.43.0" 2024-09-13T22:20:32.075 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:31 smithi114 systemd[1]: Stopping Ceph mon.c for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:33.107 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c[108283]: 2024-09-13T22:20:32.856+0000 7ff6e71ea640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:20:33.107 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:32 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c[108283]: 2024-09-13T22:20:32.856+0000 7ff6e71ea640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T22:20:34.415 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:34 smithi114 podman[121127]: 2024-09-13 22:20:34.138343267 +0000 UTC m=+1.415897506 container died 6670a4db3663d319563d861d032c4a638c4cf26f8dcbc9731c73ae85a4f3c934 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T22:20:34.815 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:34 smithi114 podman[121127]: 2024-09-13 22:20:34.596826048 +0000 UTC m=+1.874380242 container cleanup 6670a4db3663d319563d861d032c4a638c4cf26f8dcbc9731c73ae85a4f3c934 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T22:20:34.816 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:34 smithi114 bash[121127]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c 2024-09-13T22:20:35.074 INFO:journalctl@ceph.mon.c.smithi114.stdout:Sep 13 22:20:34 smithi114 podman[121172]: 2024-09-13 22:20:34.815017061 +0000 UTC m=+1.909408628 container remove 6670a4db3663d319563d861d032c4a638c4cf26f8dcbc9731c73ae85a4f3c934 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-c, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-13T22:20:36.206 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T22:20:36.249 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:20:36.250 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T22:20:36.250 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T22:20:36.250 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.b 2024-09-13T22:20:36.621 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:36 smithi121 systemd[1]: Stopping Ceph mon.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b[83509]: 2024-09-13T22:20:36.729+0000 7fe63c091640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:20:37.121 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b[83509]: 2024-09-13T22:20:36.729+0000 7fe63c091640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T22:20:37.122 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:36 smithi121 podman[90058]: 2024-09-13 22:20:36.92438522 +0000 UTC m=+0.465521934 container died 2847ae29281d81fbd59b91a282062e41a951fe966b5a38337d7cc8a5b14757e1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T22:20:37.558 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:37 smithi121 podman[90058]: 2024-09-13 22:20:37.355883471 +0000 UTC m=+0.897020182 container cleanup 2847ae29281d81fbd59b91a282062e41a951fe966b5a38337d7cc8a5b14757e1 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T22:20:37.558 INFO:journalctl@ceph.mon.b.smithi121.stdout:Sep 13 22:20:37 smithi121 bash[90058]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mon-b 2024-09-13T22:20:37.778 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T22:20:37.830 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:20:37.830 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T22:20:37.830 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T22:20:37.830 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y 2024-09-13T22:20:38.157 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:37 smithi114 systemd[1]: Stopping Ceph mgr.y for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:38.440 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:38 smithi114 podman[121638]: 2024-09-13 22:20:38.164192205 +0000 UTC m=+0.143578367 container died 8cc35749ec26708bdc9c4dd9d2af19c0f76b8ab8a75d3040599ec5c43f49d125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T22:20:38.799 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:38 smithi114 podman[121638]: 2024-09-13 22:20:38.597984056 +0000 UTC m=+0.577370247 container cleanup 8cc35749ec26708bdc9c4dd9d2af19c0f76b8ab8a75d3040599ec5c43f49d125 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T22:20:38.800 INFO:journalctl@ceph.mgr.y.smithi114.stdout:Sep 13 22:20:38 smithi114 bash[121638]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-y 2024-09-13T22:20:39.021 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T22:20:39.060 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:20:39.061 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T22:20:39.061 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T22:20:39.061 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x 2024-09-13T22:20:39.371 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:20:39 smithi121 systemd[1]: Stopping Ceph mgr.x for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:39.788 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:20:39 smithi121 podman[90243]: 2024-09-13 22:20:39.508337171 +0000 UTC m=+0.244563525 container died 9dc33ce81ccf0c4e40d8c9b960a342d7719e6ae8a4ebe34b65794cb93eb38f1b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:20:40.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:20:39 smithi121 podman[90243]: 2024-09-13 22:20:39.978135229 +0000 UTC m=+0.714361582 container cleanup 9dc33ce81ccf0c4e40d8c9b960a342d7719e6ae8a4ebe34b65794cb93eb38f1b (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T22:20:40.121 INFO:journalctl@ceph.mgr.x.smithi121.stdout:Sep 13 22:20:39 smithi121 bash[90243]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-mgr-x 2024-09-13T22:20:40.350 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T22:20:40.388 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:20:40.388 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T22:20:40.389 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T22:20:40.389 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.0 2024-09-13T22:20:40.720 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:40 smithi114 systemd[1]: Stopping Ceph osd.0 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:41.075 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[117317]: 2024-09-13T22:20:40.719+0000 7f25a9a09640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:20:41.076 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[117317]: 2024-09-13T22:20:40.719+0000 7f25a9a09640 -1 osd.0 123 *** Got signal Terminated *** 2024-09-13T22:20:41.076 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:40 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0[117317]: 2024-09-13T22:20:40.719+0000 7f25a9a09640 -1 osd.0 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:20:44.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:20:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:20:44.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:20:44.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:20:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:20:44.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:20:46.575 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:46 smithi114 podman[121798]: 2024-09-13 22:20:46.178702692 +0000 UTC m=+5.597792400 container died 4efb123ca48e990e07823830cd6ade7673ba10ba5251f064b27e2b8ad4efbc62 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T22:20:47.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:46 smithi114 podman[121798]: 2024-09-13 22:20:46.627053961 +0000 UTC m=+6.046143667 container cleanup 4efb123ca48e990e07823830cd6ade7673ba10ba5251f064b27e2b8ad4efbc62 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T22:20:47.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:46 smithi114 bash[121798]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0 2024-09-13T22:20:47.074 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:46 smithi114 podman[121814]: 2024-09-13 22:20:46.803234685 +0000 UTC m=+0.617283195 container remove 4efb123ca48e990e07823830cd6ade7673ba10ba5251f064b27e2b8ad4efbc62 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:20:47.548 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:47 smithi114 podman[121904]: 2024-09-13 22:20:47.14644489 +0000 UTC m=+0.015747859 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:20:47.548 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:47 smithi114 podman[121904]: 2024-09-13 22:20:47.253933307 +0000 UTC m=+0.123236261 container create 74afead6d43a50da4bbd80c713ca2f10e9b33609b5d5b931848a092162afcee6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T22:20:47.548 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:47 smithi114 podman[121904]: 2024-09-13 22:20:47.489656758 +0000 UTC m=+0.358959713 container init 74afead6d43a50da4bbd80c713ca2f10e9b33609b5d5b931848a092162afcee6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:20:47.549 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:47 smithi114 podman[121904]: 2024-09-13 22:20:47.493311918 +0000 UTC m=+0.362614879 container start 74afead6d43a50da4bbd80c713ca2f10e9b33609b5d5b931848a092162afcee6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T22:20:47.549 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:47 smithi114 podman[121904]: 2024-09-13 22:20:47.547800844 +0000 UTC m=+0.417103811 container attach 74afead6d43a50da4bbd80c713ca2f10e9b33609b5d5b931848a092162afcee6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T22:20:47.824 INFO:journalctl@ceph.osd.0.smithi114.stdout:Sep 13 22:20:47 smithi114 podman[121904]: 2024-09-13 22:20:47.680679899 +0000 UTC m=+0.549982865 container died 74afead6d43a50da4bbd80c713ca2f10e9b33609b5d5b931848a092162afcee6 (image=quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T22:20:48.399 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T22:20:48.439 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:20:48.439 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T22:20:48.440 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T22:20:48.440 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.1 2024-09-13T22:20:48.639 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:48 smithi114 systemd[1]: Stopping Ceph osd.1 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:49.074 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1[34730]: 2024-09-13T22:20:48.637+0000 7f078d433700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:20:49.074 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1[34730]: 2024-09-13T22:20:48.637+0000 7f078d433700 -1 osd.1 123 *** Got signal Terminated *** 2024-09-13T22:20:49.074 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:48 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1[34730]: 2024-09-13T22:20:48.637+0000 7f078d433700 -1 osd.1 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:20:54.235 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:53 smithi114 podman[121981]: 2024-09-13 22:20:53.964322728 +0000 UTC m=+5.439345216 container died 4c38b33df90e675479f125cc143e81feeeaa8e06b0732cfa017d27d1a469d4ec (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, version=8, io.openshift.expose-services=, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.component=centos-stream-container) 2024-09-13T22:20:54.498 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:54 smithi114 podman[121981]: 2024-09-13 22:20:54.459578375 +0000 UTC m=+5.934600852 container cleanup 4c38b33df90e675479f125cc143e81feeeaa8e06b0732cfa017d27d1a469d4ec (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , version=8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T22:20:54.498 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:54 smithi114 bash[121981]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1 2024-09-13T22:20:54.498 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:54 smithi114 bash[122418]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd.1" found: no such container 2024-09-13T22:20:54.498 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:20:54 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:20:54.244Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:20:54.499 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:20:54 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:20:54.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:20:54.499 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:20:54 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:20:54.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:20:55.056 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:54 smithi114 podman[122395]: 2024-09-13 22:20:54.727574278 +0000 UTC m=+0.762340770 container remove 4c38b33df90e675479f125cc143e81feeeaa8e06b0732cfa017d27d1a469d4ec (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1, release=754, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, distribution-scope=public, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD) 2024-09-13T22:20:55.056 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:54 smithi114 podman[122512]: 2024-09-13 22:20:54.845094979 +0000 UTC m=+0.021181008 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:20:55.056 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:54 smithi114 podman[122512]: 2024-09-13 22:20:54.961648834 +0000 UTC m=+0.137734895 container create 41bac76f9a1f2e03d6d1febf9894dc26ab72f1a55a38669f83560ec2a1a5c043 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_CLEAN=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T22:20:55.566 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:55 smithi114 podman[122512]: 2024-09-13 22:20:55.490707719 +0000 UTC m=+0.666793740 container init 41bac76f9a1f2e03d6d1febf9894dc26ab72f1a55a38669f83560ec2a1a5c043 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, ceph=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:20:55.566 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:55 smithi114 podman[122512]: 2024-09-13 22:20:55.494337065 +0000 UTC m=+0.670423089 container start 41bac76f9a1f2e03d6d1febf9894dc26ab72f1a55a38669f83560ec2a1a5c043 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1-deactivate, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, version=8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64) 2024-09-13T22:20:55.566 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:55 smithi114 podman[122512]: 2024-09-13 22:20:55.565500264 +0000 UTC m=+0.741586296 container attach 41bac76f9a1f2e03d6d1febf9894dc26ab72f1a55a38669f83560ec2a1a5c043 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1-deactivate, io.k8s.display-name=CentOS Stream 8, name=centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, version=8, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.component=centos-stream-container) 2024-09-13T22:20:56.324 INFO:journalctl@ceph.osd.1.smithi114.stdout:Sep 13 22:20:55 smithi114 podman[122512]: 2024-09-13 22:20:55.877424834 +0000 UTC m=+1.053510854 container died 41bac76f9a1f2e03d6d1febf9894dc26ab72f1a55a38669f83560ec2a1a5c043 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-1-deactivate, vcs-type=git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, release=754, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream) 2024-09-13T22:20:56.462 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T22:20:56.506 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:20:56.507 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T22:20:56.507 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T22:20:56.507 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.2 2024-09-13T22:20:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:20:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:20:56.258Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-09-13T22:20:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:20:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:20:56.261Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-09-13T22:20:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:20:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:20:56.262Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-09-13T22:20:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:20:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:20:56.262Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-09-13T22:20:56.622 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:20:56 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:20:56.263Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.114:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.114:8765: connect: connection refused" 2024-09-13T22:20:56.773 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:20:56 smithi114 systemd[1]: Stopping Ceph osd.2 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:20:57.574 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:20:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2[37723]: 2024-09-13T22:20:57.309+0000 7f1179990700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:20:57.575 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:20:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2[37723]: 2024-09-13T22:20:57.309+0000 7f1179990700 -1 osd.2 123 *** Got signal Terminated *** 2024-09-13T22:20:57.575 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:20:57 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2[37723]: 2024-09-13T22:20:57.309+0000 7f1179990700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:21:02.963 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:02 smithi114 podman[122699]: 2024-09-13 22:21:02.690653327 +0000 UTC m=+6.084216926 container died d4e942a420eaa27e7f1d209f2de91255126927ec2dbea9686620d4bd19dc6b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.expose-services=, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , version=8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T22:21:03.305 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 podman[122699]: 2024-09-13 22:21:03.12825015 +0000 UTC m=+6.521813758 container cleanup d4e942a420eaa27e7f1d209f2de91255126927ec2dbea9686620d4bd19dc6b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2, io.openshift.expose-services=, GIT_CLEAN=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-type=git, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, distribution-scope=public, ceph=True, architecture=x86_64, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870) 2024-09-13T22:21:03.305 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 bash[122699]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2 2024-09-13T22:21:03.561 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 podman[122987]: 2024-09-13 22:21:03.304524559 +0000 UTC m=+0.610321336 container remove d4e942a420eaa27e7f1d209f2de91255126927ec2dbea9686620d4bd19dc6b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.expose-services=, ceph=True, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, architecture=x86_64, io.openshift.tags=base centos centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc.) 2024-09-13T22:21:03.561 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 bash[123001]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd.2" found: no such container 2024-09-13T22:21:03.824 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 podman[123060]: 2024-09-13 22:21:03.464144855 +0000 UTC m=+0.016965783 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:21:03.825 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 podman[123060]: 2024-09-13 22:21:03.571819589 +0000 UTC m=+0.124640547 container create 88e1dcae1324bdf3d45c22cd3c7824c20fcf82cb8ce75f1270a404a2d671f0d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2-deactivate, architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, name=centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, distribution-scope=public) 2024-09-13T22:21:03.825 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 podman[123060]: 2024-09-13 22:21:03.790704568 +0000 UTC m=+0.343525483 container init 88e1dcae1324bdf3d45c22cd3c7824c20fcf82cb8ce75f1270a404a2d671f0d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_CLEAN=True, io.buildah.version=1.19.8, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.tags=base centos centos-stream, release=754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, GIT_BRANCH=HEAD) 2024-09-13T22:21:03.825 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 podman[123060]: 2024-09-13 22:21:03.79465525 +0000 UTC m=+0.347476166 container start 88e1dcae1324bdf3d45c22cd3c7824c20fcf82cb8ce75f1270a404a2d671f0d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, GIT_CLEAN=True, release=754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, version=8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870) 2024-09-13T22:21:04.077 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:03 smithi114 podman[123060]: 2024-09-13 22:21:03.874145367 +0000 UTC m=+0.426966283 container attach 88e1dcae1324bdf3d45c22cd3c7824c20fcf82cb8ce75f1270a404a2d671f0d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2-deactivate, distribution-scope=public, vendor=Red Hat, Inc., ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, release=754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T22:21:04.420 INFO:journalctl@ceph.osd.2.smithi114.stdout:Sep 13 22:21:04 smithi114 podman[123060]: 2024-09-13 22:21:04.075562356 +0000 UTC m=+0.628383281 container died 88e1dcae1324bdf3d45c22cd3c7824c20fcf82cb8ce75f1270a404a2d671f0d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-2-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_CLEAN=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.component=centos-stream-container, architecture=x86_64, RELEASE=HEAD, vcs-type=git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , version=8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8) 2024-09-13T22:21:04.421 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:04.244Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:04.421 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:04.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:21:04.421 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:04 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:04.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:04.794 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T22:21:04.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:21:04.833 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T22:21:04.834 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T22:21:04.834 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.3 2024-09-13T22:21:05.074 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:04 smithi114 systemd[1]: Stopping Ceph osd.3 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:05.074 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T22:21:05.021+0000 7fbc86cd9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:21:05.075 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T22:21:05.021+0000 7fbc86cd9700 -1 osd.3 123 *** Got signal Terminated *** 2024-09-13T22:21:05.075 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:05 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T22:21:05.021+0000 7fbc86cd9700 -1 osd.3 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:21:07.324 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:07 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T22:21:07.043+0000 7fbc83e75700 -1 osd.3 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.008239+0000 front 2024-09-13T22:20:44.008380+0000 (oldest deadline 2024-09-13T22:21:06.308344+0000) 2024-09-13T22:21:07.622 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:07.265+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:08.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:07 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:07.742+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:08.324 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:08 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T22:21:08.028+0000 7fbc83e75700 -1 osd.3 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.008239+0000 front 2024-09-13T22:20:44.008380+0000 (oldest deadline 2024-09-13T22:21:06.308344+0000) 2024-09-13T22:21:08.621 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:08.220+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:09.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:08 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:08.692+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:09.324 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:09 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T22:21:09.022+0000 7fbc83e75700 -1 osd.3 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.008239+0000 front 2024-09-13T22:20:44.008380+0000 (oldest deadline 2024-09-13T22:21:06.308344+0000) 2024-09-13T22:21:09.583 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:09.351+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:09.584 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:09.191+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:09.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:09 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:09.708+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:10.324 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:09 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3[40691]: 2024-09-13T22:21:09.980+0000 7fbc83e75700 -1 osd.3 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.008239+0000 front 2024-09-13T22:20:44.008380+0000 (oldest deadline 2024-09-13T22:21:06.308344+0000) 2024-09-13T22:21:10.621 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:10.352+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:10.622 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:10.207+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:10.732 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:10 smithi114 podman[123140]: 2024-09-13 22:21:10.454391888 +0000 UTC m=+5.527504726 container died 60269299bda81617e27fd41cf9a1324b603548e0e70e677b59b8701dc5f7ca11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, architecture=x86_64, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, name=centos-stream) 2024-09-13T22:21:11.049 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:10 smithi114 podman[123140]: 2024-09-13 22:21:10.830778195 +0000 UTC m=+5.903891022 container cleanup 60269299bda81617e27fd41cf9a1324b603548e0e70e677b59b8701dc5f7ca11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, version=8, release=754) 2024-09-13T22:21:11.049 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:10 smithi114 bash[123140]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3 2024-09-13T22:21:11.049 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:10 smithi114 bash[123227]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd.3" found: no such container 2024-09-13T22:21:11.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:10 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:10.685+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:11.324 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:11 smithi114 podman[123214]: 2024-09-13 22:21:11.048752832 +0000 UTC m=+0.593841895 container remove 60269299bda81617e27fd41cf9a1324b603548e0e70e677b59b8701dc5f7ca11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, release=754, architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, ceph=True, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, distribution-scope=public, maintainer=Guillaume Abrioux , version=8) 2024-09-13T22:21:11.516 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:11.370+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:11.516 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:11.188+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:11.678 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:11 smithi114 podman[123289]: 2024-09-13 22:21:11.294670622 +0000 UTC m=+0.017633680 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:21:11.679 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:11 smithi114 podman[123289]: 2024-09-13 22:21:11.43381027 +0000 UTC m=+0.156773318 container create baa32f6104a13e5e1cfb26f98669d41aa118aab74443e3c3662586396ba83fa7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3-deactivate, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., release=754, GIT_CLEAN=True, RELEASE=HEAD, version=8, vcs-type=git, name=centos-stream, ceph=True, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T22:21:11.679 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:11 smithi114 podman[123289]: 2024-09-13 22:21:11.677899297 +0000 UTC m=+0.400862344 container init baa32f6104a13e5e1cfb26f98669d41aa118aab74443e3c3662586396ba83fa7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vcs-type=git, architecture=x86_64) 2024-09-13T22:21:11.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:11.679+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:11.871 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:11 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:11.515+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.142258+0000 front 2024-09-13T22:20:45.142367+0000 (oldest deadline 2024-09-13T22:21:11.042146+0000) 2024-09-13T22:21:11.930 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:11 smithi114 podman[123289]: 2024-09-13 22:21:11.68142583 +0000 UTC m=+0.404388877 container start baa32f6104a13e5e1cfb26f98669d41aa118aab74443e3c3662586396ba83fa7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-type=git, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, release=754, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8) 2024-09-13T22:21:11.930 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:11 smithi114 podman[123289]: 2024-09-13 22:21:11.769444101 +0000 UTC m=+0.492407157 container attach baa32f6104a13e5e1cfb26f98669d41aa118aab74443e3c3662586396ba83fa7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8) 2024-09-13T22:21:12.563 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:12 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:12.386+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:12.564 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:12 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:12.213+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:12.824 INFO:journalctl@ceph.osd.3.smithi114.stdout:Sep 13 22:21:12 smithi114 podman[123289]: 2024-09-13 22:21:12.611479023 +0000 UTC m=+1.334442070 container died baa32f6104a13e5e1cfb26f98669d41aa118aab74443e3c3662586396ba83fa7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-3-deactivate, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-type=git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T22:21:12.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:12 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:12.704+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:12.871 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:12 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:12.562+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.142258+0000 front 2024-09-13T22:20:45.142367+0000 (oldest deadline 2024-09-13T22:21:11.042146+0000) 2024-09-13T22:21:13.184 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T22:21:13.231 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:21:13.231 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T22:21:13.231 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T22:21:13.231 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.4 2024-09-13T22:21:13.600 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:13.346+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:13.600 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:13.195+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:13.600 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:13 smithi121 systemd[1]: Stopping Ceph osd.4 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:13.601 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:13.424+0000 7f549d0c2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:21:13.601 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:13.424+0000 7f549d0c2700 -1 osd.4 123 *** Got signal Terminated *** 2024-09-13T22:21:13.601 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:13.424+0000 7f549d0c2700 -1 osd.4 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:21:13.871 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:13.599+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.142258+0000 front 2024-09-13T22:20:45.142367+0000 (oldest deadline 2024-09-13T22:21:11.042146+0000) 2024-09-13T22:21:13.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:13 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:13.680+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:14.426 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:14 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:14.245Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:14.426 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:14 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:14.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:21:14.426 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:14 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:14.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:14.564 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:14.301+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:14.565 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:14.244+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:14.565 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:14.244+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:14.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:14.683+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:14.871 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:14 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:14.563+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.142258+0000 front 2024-09-13T22:20:45.142367+0000 (oldest deadline 2024-09-13T22:21:11.042146+0000) 2024-09-13T22:21:15.555 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:15 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:15.265+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:15.555 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:15 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:15.265+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:15.556 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:15 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:15.285+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:15.556 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:15 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:15.285+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:15.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:15 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:15.677+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:15.871 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:15 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:15.554+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.142258+0000 front 2024-09-13T22:20:45.142367+0000 (oldest deadline 2024-09-13T22:21:11.042146+0000) 2024-09-13T22:21:16.555 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:16.307+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:16.555 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:16.307+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:16.555 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:16.218+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:16.555 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:16.218+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:16.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:16.678+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:16.872 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:16.553+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.142258+0000 front 2024-09-13T22:20:45.142367+0000 (oldest deadline 2024-09-13T22:21:11.042146+0000) 2024-09-13T22:21:16.872 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:16 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:16.553+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:52.742881+0000 front 2024-09-13T22:20:52.742934+0000 (oldest deadline 2024-09-13T22:21:16.242940+0000) 2024-09-13T22:21:17.550 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:17 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:17.333+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:17.550 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:17 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:17.333+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:17.551 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:17 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:17.220+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:17.551 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:17 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:17.220+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:17.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:17 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:17.699+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:17.871 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:17 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:17.549+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.142258+0000 front 2024-09-13T22:20:45.142367+0000 (oldest deadline 2024-09-13T22:21:11.042146+0000) 2024-09-13T22:21:17.872 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:17 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4[29504]: 2024-09-13T22:21:17.549+0000 7f549aa5f700 -1 osd.4 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:52.742881+0000 front 2024-09-13T22:20:52.742934+0000 (oldest deadline 2024-09-13T22:21:16.242940+0000) 2024-09-13T22:21:18.621 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:18.370+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:18.621 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:18.370+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:18.622 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:18.264+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:18.622 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:18.264+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:19.109 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:18 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:18.684+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:19.109 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:18 smithi121 podman[91200]: 2024-09-13 22:21:18.828065683 +0000 UTC m=+5.524413819 container died 4e71e78263ddbcbf8cf37dadd7d6744a3763fe99266b7466bb47a630bfb602a9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4, distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=HEAD) 2024-09-13T22:21:19.371 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:19 smithi121 podman[91200]: 2024-09-13 22:21:19.273991067 +0000 UTC m=+5.970339267 container cleanup 4e71e78263ddbcbf8cf37dadd7d6744a3763fe99266b7466bb47a630bfb602a9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, distribution-scope=public, io.buildah.version=1.19.8, vcs-type=git, release=754, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, version=8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:21:19.371 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:19 smithi121 bash[91200]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4 2024-09-13T22:21:19.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:19 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:19.229+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:19.372 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:19 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:19.229+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:19.699 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:19 smithi121 podman[91214]: 2024-09-13 22:21:19.483910859 +0000 UTC m=+0.645216708 container remove 4e71e78263ddbcbf8cf37dadd7d6744a3763fe99266b7466bb47a630bfb602a9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , release=754, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, version=8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.tags=base centos centos-stream) 2024-09-13T22:21:19.699 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:19 smithi121 bash[91231]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd.4" found: no such container 2024-09-13T22:21:19.700 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:19 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:19.379+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:19.700 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:19 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:19.379+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:19.956 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:19 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:19.697+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:19.956 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:19 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:19.697+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:19.957 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:19 smithi121 podman[91332]: 2024-09-13 22:21:19.665436515 +0000 UTC m=+0.017241108 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:21:19.957 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:19 smithi121 podman[91332]: 2024-09-13 22:21:19.809899521 +0000 UTC m=+0.161704101 container create 4d4007d8f841d84b49320ccf94c29194198ebac5c2ed7762cfafb589a23c963b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, name=centos-stream, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, RELEASE=HEAD, ceph=True, vcs-type=git) 2024-09-13T22:21:20.247 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:20 smithi121 podman[91332]: 2024-09-13 22:21:20.062762113 +0000 UTC m=+0.414566696 container init 4d4007d8f841d84b49320ccf94c29194198ebac5c2ed7762cfafb589a23c963b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream) 2024-09-13T22:21:20.247 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:20 smithi121 podman[91332]: 2024-09-13 22:21:20.066127072 +0000 UTC m=+0.417931658 container start 4d4007d8f841d84b49320ccf94c29194198ebac5c2ed7762cfafb589a23c963b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4-deactivate, com.redhat.component=centos-stream-container, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, version=8, io.openshift.tags=base centos centos-stream, ceph=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8) 2024-09-13T22:21:20.248 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:20 smithi121 podman[91332]: 2024-09-13 22:21:20.145955239 +0000 UTC m=+0.497759822 container attach 4d4007d8f841d84b49320ccf94c29194198ebac5c2ed7762cfafb589a23c963b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4-deactivate, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, architecture=x86_64, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, ceph=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, name=centos-stream) 2024-09-13T22:21:20.621 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:20.414+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:20.621 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:20.414+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:20.622 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:20 smithi121 podman[91332]: 2024-09-13 22:21:20.377379043 +0000 UTC m=+0.729183634 container died 4d4007d8f841d84b49320ccf94c29194198ebac5c2ed7762cfafb589a23c963b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T22:21:20.622 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:20.245+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:20.622 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:20.245+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:20.944 INFO:journalctl@ceph.osd.4.smithi121.stdout:Sep 13 22:21:20 smithi121 podman[91451]: 2024-09-13 22:21:20.941069177 +0000 UTC m=+0.554299489 container remove 4d4007d8f841d84b49320ccf94c29194198ebac5c2ed7762cfafb589a23c963b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-type=git, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T22:21:20.944 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:20.748+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:20.944 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:20 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:20.748+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:20.968 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T22:21:21.007 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:21:21.008 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T22:21:21.008 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T22:21:21.008 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.5 2024-09-13T22:21:21.216 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:21 smithi121 systemd[1]: Stopping Ceph osd.5 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:21.216 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:21.214+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:21.216 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:21.214+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:21.503 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:21.239+0000 7fc8b0b8c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:21:21.503 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:21.239+0000 7fc8b0b8c700 -1 osd.5 123 *** Got signal Terminated *** 2024-09-13T22:21:21.504 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:21.239+0000 7fc8b0b8c700 -1 osd.5 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:21:21.504 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:21.433+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:21.504 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:21.433+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:22.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:21.759+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:22.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:21 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:21.759+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:22.387 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:22.187+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:22.387 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:22.187+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:22.711 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:22.463+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:22.711 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:22.463+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:22.994 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:22.710+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:22.995 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:22 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:22.710+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:23.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:23.163+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:23.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:23.163+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:23.757 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:23.449+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:23.757 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:23.449+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:24.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:23.756+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:24.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:23 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:23.756+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:24.456 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:24.160+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:24.456 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:24.160+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:24.456 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:24.160+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:24.573 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:24.245Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:21:24.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:24.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:21:24.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:24 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:24.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:24.715 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:24.464+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:24.716 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:24.464+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:25.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:24.715+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:25.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:24 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:24.715+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:25.480 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:25.173+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:25.480 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:25.173+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:25.480 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:25.173+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:25.830 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:25.478+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.470014+0000 front 2024-09-13T22:20:44.469957+0000 (oldest deadline 2024-09-13T22:21:09.169962+0000) 2024-09-13T22:21:25.830 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5[32411]: 2024-09-13T22:21:25.478+0000 7fc8ae529700 -1 osd.5 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.170395+0000 front 2024-09-13T22:20:49.170404+0000 (oldest deadline 2024-09-13T22:21:14.470400+0000) 2024-09-13T22:21:25.830 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:25.723+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:25.830 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:25.723+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:25.830 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:25 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:25.723+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:26.481 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:26.143+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:26.481 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:26.143+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:26.481 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:26.143+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:26.871 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:26 smithi121 podman[91561]: 2024-09-13 22:21:26.573170327 +0000 UTC m=+5.470779466 container died 95e89c636120dcc88b38d18716b235aef3dff51190ba7a147b3ca918ca3dbc11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, RELEASE=HEAD, ceph=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux ) 2024-09-13T22:21:26.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:26.718+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:26.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:26.718+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:26.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:26 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:26.718+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:27.145 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:26 smithi121 podman[91561]: 2024-09-13 22:21:26.98638375 +0000 UTC m=+5.883992886 container cleanup 95e89c636120dcc88b38d18716b235aef3dff51190ba7a147b3ca918ca3dbc11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, GIT_CLEAN=True) 2024-09-13T22:21:27.145 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:26 smithi121 bash[91561]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5 2024-09-13T22:21:27.540 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:27.144+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:27.540 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:27.144+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:27.540 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:27.144+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:27.817 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:27 smithi121 bash[92150]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd.5" found: no such container 2024-09-13T22:21:27.817 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:27.715+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:27.817 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:27.715+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:27.817 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:27 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:27.715+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:28.121 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:27 smithi121 podman[92099]: 2024-09-13 22:21:27.816209839 +0000 UTC m=+1.237639238 container remove 95e89c636120dcc88b38d18716b235aef3dff51190ba7a147b3ca918ca3dbc11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, release=754, distribution-scope=public, vcs-type=git, GIT_CLEAN=True, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T22:21:28.121 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:28 smithi121 podman[92212]: 2024-09-13 22:21:28.008658103 +0000 UTC m=+0.107067557 container create 78bbccc4a61d064339de7869eef22a30437ecfb6b289356f7a95fc9a1d8a971d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, version=8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, GIT_BRANCH=HEAD, release=754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-type=git, com.redhat.component=centos-stream-container, name=centos-stream, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, ceph=True) 2024-09-13T22:21:28.121 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:28 smithi121 podman[92212]: 2024-09-13 22:21:27.917956596 +0000 UTC m=+0.016366058 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:21:28.529 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:28.132+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:28.529 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:28.132+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:28.529 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:28.132+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:28.530 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:28 smithi121 podman[92212]: 2024-09-13 22:21:28.244508086 +0000 UTC m=+0.342917549 container init 78bbccc4a61d064339de7869eef22a30437ecfb6b289356f7a95fc9a1d8a971d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5-deactivate, io.openshift.expose-services=, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.openshift.tags=base centos centos-stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vendor=Red Hat, Inc., version=8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T22:21:28.530 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:28 smithi121 podman[92212]: 2024-09-13 22:21:28.24785313 +0000 UTC m=+0.346262588 container start 78bbccc4a61d064339de7869eef22a30437ecfb6b289356f7a95fc9a1d8a971d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, name=centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:21:28.530 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:28 smithi121 podman[92212]: 2024-09-13 22:21:28.319595214 +0000 UTC m=+0.418004682 container attach 78bbccc4a61d064339de7869eef22a30437ecfb6b289356f7a95fc9a1d8a971d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-type=git, io.openshift.expose-services=, com.redhat.component=centos-stream-container, RELEASE=HEAD, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, version=8) 2024-09-13T22:21:28.530 INFO:journalctl@ceph.osd.5.smithi121.stdout:Sep 13 22:21:28 smithi121 podman[92212]: 2024-09-13 22:21:28.528496546 +0000 UTC m=+0.626906004 container died 78bbccc4a61d064339de7869eef22a30437ecfb6b289356f7a95fc9a1d8a971d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-type=git, version=8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.buildah.version=1.19.8) 2024-09-13T22:21:28.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:28.678+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:28.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:28.678+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:28.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:28 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:28.678+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:29.365 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:29.151+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:29.366 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:29.151+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:29.366 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:29.151+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:29.401 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T22:21:29.446 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:21:29.446 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T22:21:29.447 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T22:21:29.447 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.6 2024-09-13T22:21:29.621 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:29 smithi121 systemd[1]: Stopping Ceph osd.6 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:30.121 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:29.668+0000 7f9cb19d4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:21:30.122 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:29.668+0000 7f9cb19d4700 -1 osd.6 123 *** Got signal Terminated *** 2024-09-13T22:21:30.122 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:29.668+0000 7f9cb19d4700 -1 osd.6 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:21:30.122 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:29.639+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:30.122 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:29.639+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:30.122 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:29 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:29.639+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:30.595 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:30.149+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:30.596 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:30.149+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:30.596 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:30.149+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:30.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:30.594+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:30.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:30.594+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:30.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:30 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:30.594+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:31.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:31.100+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:31.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:31.100+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:31.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:31.100+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:31.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:31.600+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:31.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:31.600+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:31.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:31.600+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:31.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:31 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:31.600+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:32.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:32.077+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:32.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:32.077+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:32.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:32.077+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:32.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:32.619+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:32.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:32.619+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:32.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:32.619+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:32.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:32 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:32.619+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:33.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:33.054+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:33.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:33.054+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:33.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:33.054+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:33.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:33.054+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.398523+0000 front 2024-09-13T22:21:09.398579+0000 (oldest deadline 2024-09-13T22:21:32.898401+0000) 2024-09-13T22:21:33.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:33.582+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:33.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:33.582+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:33.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:33.582+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:33.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:33 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:33.582+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:34.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:34.044+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:45.395683+0000 front 2024-09-13T22:20:45.395597+0000 (oldest deadline 2024-09-13T22:21:07.095477+0000) 2024-09-13T22:21:34.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:34.044+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:49.996460+0000 front 2024-09-13T22:20:49.996468+0000 (oldest deadline 2024-09-13T22:21:14.096488+0000) 2024-09-13T22:21:34.371 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:34.044+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:20:59.397666+0000 front 2024-09-13T22:20:59.397616+0000 (oldest deadline 2024-09-13T22:21:23.497677+0000) 2024-09-13T22:21:34.372 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6[35372]: 2024-09-13T22:21:34.044+0000 7f9caab68700 -1 osd.6 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.398523+0000 front 2024-09-13T22:21:09.398579+0000 (oldest deadline 2024-09-13T22:21:32.898401+0000) 2024-09-13T22:21:34.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:34.246Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:34.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:34.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:21:34.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:34 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:34.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:34.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:34.560+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:34.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:34.560+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:34.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:34.560+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:34.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:34 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:34.560+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:35.453 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:35 smithi121 podman[92290]: 2024-09-13 22:21:35.171243578 +0000 UTC m=+5.633293781 container died 369b1a1170edb83061014ec107dd9a61a152e726698e7c53a47026916e223e8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux ) 2024-09-13T22:21:35.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:35.592+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:35.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:35.592+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:35.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:35.592+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:35.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:35 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:35.592+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:35.872 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:35 smithi121 podman[92290]: 2024-09-13 22:21:35.626012807 +0000 UTC m=+6.088063051 container cleanup 369b1a1170edb83061014ec107dd9a61a152e726698e7c53a47026916e223e8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, version=8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=centos-stream-container, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public) 2024-09-13T22:21:35.872 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:35 smithi121 bash[92290]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6 2024-09-13T22:21:35.872 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:35 smithi121 bash[92319]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd.6" found: no such container 2024-09-13T22:21:36.166 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:35 smithi121 podman[92305]: 2024-09-13 22:21:35.910761566 +0000 UTC m=+0.736132843 container remove 369b1a1170edb83061014ec107dd9a61a152e726698e7c53a47026916e223e8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, version=8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, release=754, architecture=x86_64, com.redhat.component=centos-stream-container) 2024-09-13T22:21:36.166 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:36 smithi121 podman[92380]: 2024-09-13 22:21:36.036709662 +0000 UTC m=+0.090164410 container create 26918f048c29bb1c06873d757a7fc9502384fe3edc81ec1371ae78b45cce835f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, architecture=x86_64, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vcs-type=git) 2024-09-13T22:21:36.167 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:36 smithi121 podman[92380]: 2024-09-13 22:21:35.963946807 +0000 UTC m=+0.017401525 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:21:36.544 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:36 smithi121 podman[92380]: 2024-09-13 22:21:36.255584686 +0000 UTC m=+0.309039392 container init 26918f048c29bb1c06873d757a7fc9502384fe3edc81ec1371ae78b45cce835f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vendor=Red Hat, Inc., ceph=True, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, RELEASE=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, distribution-scope=public, version=8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-09-13T22:21:36.544 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:36 smithi121 podman[92380]: 2024-09-13 22:21:36.259066376 +0000 UTC m=+0.312521088 container start 26918f048c29bb1c06873d757a7fc9502384fe3edc81ec1371ae78b45cce835f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6-deactivate, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, architecture=x86_64, distribution-scope=public, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870) 2024-09-13T22:21:36.544 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:36 smithi121 podman[92380]: 2024-09-13 22:21:36.33890444 +0000 UTC m=+0.392359174 container attach 26918f048c29bb1c06873d757a7fc9502384fe3edc81ec1371ae78b45cce835f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6-deactivate, vcs-type=git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.openshift.expose-services=, distribution-scope=public, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, version=8) 2024-09-13T22:21:36.545 INFO:journalctl@ceph.osd.6.smithi121.stdout:Sep 13 22:21:36 smithi121 podman[92380]: 2024-09-13 22:21:36.542946375 +0000 UTC m=+0.596401087 container died 26918f048c29bb1c06873d757a7fc9502384fe3edc81ec1371ae78b45cce835f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, release=754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T22:21:36.842 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:36.628+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:36.842 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:36.628+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:36.843 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:36.628+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:36.843 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:36 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:36.628+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:37.193 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T22:21:37.231 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:21:37.232 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T22:21:37.232 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T22:21:37.232 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.7 2024-09-13T22:21:37.621 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 systemd[1]: Stopping Ceph osd.7 for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:37.621 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:37.402+0000 7f94ef723700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:21:37.622 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:37.402+0000 7f94ef723700 -1 osd.7 123 *** Got signal Terminated *** 2024-09-13T22:21:37.622 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:37.402+0000 7f94ef723700 -1 osd.7 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:21:38.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:37.657+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:38.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:37.657+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:38.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:37.657+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:38.122 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:37 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:37.657+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:39.023 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:38.662+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:39.023 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:38.662+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:39.023 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:38.662+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:39.024 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:38 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:38.662+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:39.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:39 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:39.648+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:39.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:39 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:39.648+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:39.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:39 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:39.648+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:39.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:39 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:39.648+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:40.871 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:40 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:40.616+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:40.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:40 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:40.616+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:40.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:40 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:40.616+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:40.872 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:40 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:40.616+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:42.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:41 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:41.626+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6806 osd.0 since back 2024-09-13T22:20:44.123396+0000 front 2024-09-13T22:20:44.123331+0000 (oldest deadline 2024-09-13T22:21:07.023319+0000) 2024-09-13T22:21:42.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:41 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:41.626+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6814 osd.1 since back 2024-09-13T22:20:53.424811+0000 front 2024-09-13T22:20:53.424681+0000 (oldest deadline 2024-09-13T22:21:18.724752+0000) 2024-09-13T22:21:42.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:41 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:41.626+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6822 osd.2 since back 2024-09-13T22:21:02.225651+0000 front 2024-09-13T22:21:02.225633+0000 (oldest deadline 2024-09-13T22:21:25.125540+0000) 2024-09-13T22:21:42.122 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:41 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7[38290]: 2024-09-13T22:21:41.626+0000 7f94ea8bb700 -1 osd.7 123 heartbeat_check: no reply from 172.21.15.114:6830 osd.3 since back 2024-09-13T22:21:09.826325+0000 front 2024-09-13T22:21:09.826293+0000 (oldest deadline 2024-09-13T22:21:31.526227+0000) 2024-09-13T22:21:43.121 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:42 smithi121 podman[92459]: 2024-09-13 22:21:42.807973588 +0000 UTC m=+5.489750885 container died c365a04abea491c2786f6f92622bb12be8ee05273dae57d7f625bd2315135243 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux , release=754, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True) 2024-09-13T22:21:43.492 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 podman[92459]: 2024-09-13 22:21:43.195010291 +0000 UTC m=+5.876787632 container cleanup c365a04abea491c2786f6f92622bb12be8ee05273dae57d7f625bd2315135243 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, vcs-type=git, io.openshift.expose-services=, maintainer=Guillaume Abrioux , RELEASE=HEAD, ceph=True, io.buildah.version=1.19.8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64) 2024-09-13T22:21:43.492 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 bash[92459]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7 2024-09-13T22:21:43.492 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 bash[92499]: Error: no container with name or ID "ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd.7" found: no such container 2024-09-13T22:21:43.493 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 podman[92488]: 2024-09-13 22:21:43.429571247 +0000 UTC m=+0.615987141 container remove c365a04abea491c2786f6f92622bb12be8ee05273dae57d7f625bd2315135243 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, release=754, version=8, vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, name=centos-stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:21:43.807 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 podman[92586]: 2024-09-13 22:21:43.60530681 +0000 UTC m=+0.111840413 container create b9890814c8e2a639300f1be0d4fb568a1e4ed9859e0a79a5bec0e274b2767787 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, architecture=x86_64, vcs-type=git, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870) 2024-09-13T22:21:43.808 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 podman[92586]: 2024-09-13 22:21:43.511132258 +0000 UTC m=+0.017665877 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:21:43.808 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 podman[92586]: 2024-09-13 22:21:43.807441588 +0000 UTC m=+0.313975191 container init b9890814c8e2a639300f1be0d4fb568a1e4ed9859e0a79a5bec0e274b2767787 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, release=754, io.openshift.tags=base centos centos-stream, ceph=True, name=centos-stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public) 2024-09-13T22:21:44.116 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 podman[92586]: 2024-09-13 22:21:43.810671471 +0000 UTC m=+0.317205079 container start b9890814c8e2a639300f1be0d4fb568a1e4ed9859e0a79a5bec0e274b2767787 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7-deactivate, com.redhat.component=centos-stream-container, distribution-scope=public, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream) 2024-09-13T22:21:44.117 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:43 smithi121 podman[92586]: 2024-09-13 22:21:43.890806653 +0000 UTC m=+0.397340256 container attach b9890814c8e2a639300f1be0d4fb568a1e4ed9859e0a79a5bec0e274b2767787 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7-deactivate, io.openshift.expose-services=, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vcs-type=git, version=8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, ceph=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream) 2024-09-13T22:21:44.371 INFO:journalctl@ceph.osd.7.smithi121.stdout:Sep 13 22:21:44 smithi121 podman[92586]: 2024-09-13 22:21:44.116331394 +0000 UTC m=+0.622865009 container died b9890814c8e2a639300f1be0d4fb568a1e4ed9859e0a79a5bec0e274b2767787 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-osd-7-deactivate, vendor=Red Hat, Inc., name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-type=git, io.buildah.version=1.19.8, architecture=x86_64, release=754, maintainer=Guillaume Abrioux , version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T22:21:44.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:44.247Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:44.574 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:44.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi114.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.114:8443: connect: connection refused" 2024-09-13T22:21:44.575 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:44 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:44.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi121.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.121:8443: connect: connection refused" 2024-09-13T22:21:44.764 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T22:21:44.804 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:21:44.804 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T22:21:44.805 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T22:21:44.805 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T22:21:45.017 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:44 smithi121 systemd[1]: Stopping Ceph prometheus.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:46.572 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T22:21:46.572 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T22:21:46.572 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T22:21:46.572 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.397Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.398Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.398Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T22:21:46.573 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a[71242]: ts=2024-09-13T22:21:46.398Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T22:21:46.871 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 podman[92849]: 2024-09-13 22:21:46.586673728 +0000 UTC m=+0.917345613 container died 7738fb682bbc3a159119d019f643a6f977b35f69915455eaed7f14dd3c2fa79c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:47.212 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 podman[92849]: 2024-09-13 22:21:46.956525575 +0000 UTC m=+1.287197446 container cleanup 7738fb682bbc3a159119d019f643a6f977b35f69915455eaed7f14dd3c2fa79c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:47.213 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:46 smithi121 bash[92849]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T22:21:47.621 INFO:journalctl@ceph.prometheus.a.smithi121.stdout:Sep 13 22:21:47 smithi121 podman[92901]: 2024-09-13 22:21:47.34205281 +0000 UTC m=+0.936387320 container remove 7738fb682bbc3a159119d019f643a6f977b35f69915455eaed7f14dd3c2fa79c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:48.006 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T22:21:48.048 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:21:48.049 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T22:21:48.049 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T22:21:50.016 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:49 smithi114 systemd[1]: Stopping Ceph node-exporter.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:50.279 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 podman[124654]: 2024-09-13 22:21:50.014518167 +0000 UTC m=+0.197130374 container died 43f14e3df9a396aab12500a970c99aa60ec8cc1ed61d93d037731f88417c80e6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:50.574 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 podman[124654]: 2024-09-13 22:21:50.470696895 +0000 UTC m=+0.653309096 container cleanup 43f14e3df9a396aab12500a970c99aa60ec8cc1ed61d93d037731f88417c80e6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:50.574 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 bash[124654]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T22:21:51.068 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 podman[124667]: 2024-09-13 22:21:50.755232335 +0000 UTC m=+0.744038632 container remove 43f14e3df9a396aab12500a970c99aa60ec8cc1ed61d93d037731f88417c80e6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:51.068 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:21:51.069 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T22:21:51.069 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 systemd[1]: Stopped Ceph node-exporter.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:21:51.069 INFO:journalctl@ceph.node-exporter.a.smithi114.stdout:Sep 13 22:21:50 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 3.707s CPU time. 2024-09-13T22:21:51.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:51 smithi114 systemd[1]: Stopping Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:21:51.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:51 smithi114 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a[64324]: ts=2024-09-13T22:21:51.317Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T22:21:51.634 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:51 smithi114 podman[124778]: 2024-09-13 22:21:51.368421837 +0000 UTC m=+0.162531563 container died 4b3ccb3d39923944e11cf8ad2f80a3897b00ddedb1241e11713b56c726e38109 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:51.977 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:51 smithi114 podman[124778]: 2024-09-13 22:21:51.725617282 +0000 UTC m=+0.519727016 container cleanup 4b3ccb3d39923944e11cf8ad2f80a3897b00ddedb1241e11713b56c726e38109 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:51.977 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:51 smithi114 bash[124778]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T22:21:52.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:51 smithi114 podman[124793]: 2024-09-13 22:21:51.976650111 +0000 UTC m=+0.601413450 container remove 4b3ccb3d39923944e11cf8ad2f80a3897b00ddedb1241e11713b56c726e38109 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T22:21:52.324 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:52 smithi114 podman[124793]: 2024-09-13 22:21:52.043945917 +0000 UTC m=+0.668709210 volume remove 305aa02b1e5be7be82386307ede5f16123a9a0ee58d6948bb3eb576a2486688a 2024-09-13T22:21:52.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:52 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T22:21:52.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:52 smithi114 systemd[1]: Stopped Ceph alertmanager.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:21:52.325 INFO:journalctl@ceph.alertmanager.a.smithi114.stdout:Sep 13 22:21:52 smithi114 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.950s CPU time. 2024-09-13T22:22:03.780 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T22:22:05.621 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:05 smithi121 systemd[1]: Stopping Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:22:05.939 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:05 smithi121 podman[93817]: 2024-09-13 22:22:05.624259186 +0000 UTC m=+0.108646263 container died fa1cf72b5089bf218e0fcf8d58eb18b1faa6e097ba0cee504d0d9ff539003697 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:22:06.332 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:06 smithi121 podman[93817]: 2024-09-13 22:22:06.105544324 +0000 UTC m=+0.589931443 container cleanup fa1cf72b5089bf218e0fcf8d58eb18b1faa6e097ba0cee504d0d9ff539003697 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:22:06.332 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:06 smithi121 bash[93817]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T22:22:06.621 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:06 smithi121 podman[93829]: 2024-09-13 22:22:06.331493211 +0000 UTC m=+0.708322723 container remove fa1cf72b5089bf218e0fcf8d58eb18b1faa6e097ba0cee504d0d9ff539003697 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T22:22:06.621 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:06 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T22:22:06.621 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:06 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T22:22:06.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:06 smithi121 systemd[1]: Stopped Ceph node-exporter.b for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:22:06.622 INFO:journalctl@ceph.node-exporter.b.smithi121.stdout:Sep 13 22:22:06 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 3.620s CPU time. 2024-09-13T22:22:07.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:06 smithi121 systemd[1]: Stopping Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968... 2024-09-13T22:22:07.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=server t=2024-09-13T22:22:06.87691726Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-13T22:22:07.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=tracing t=2024-09-13T22:22:06.877292551Z level=info msg="Closing tracing" 2024-09-13T22:22:07.121 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:06 smithi121 ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a[79506]: logger=ticker t=2024-09-13T22:22:06.877307981Z level=info msg=stopped last_tick=2024-09-13T22:22:00Z 2024-09-13T22:22:07.122 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:06 smithi121 podman[93938]: 2024-09-13 22:22:06.887195526 +0000 UTC m=+0.094451221 container died 6ee3553a890cd6f14da8c1ebbee2f430d96849a5a54366051c38beb73dc5d50e (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:22:07.385 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:07 smithi121 podman[93938]: 2024-09-13 22:22:07.200922463 +0000 UTC m=+0.408178151 container cleanup 6ee3553a890cd6f14da8c1ebbee2f430d96849a5a54366051c38beb73dc5d50e (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:22:07.386 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:07 smithi121 bash[93938]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a 2024-09-13T22:22:07.785 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:07 smithi121 podman[93951]: 2024-09-13 22:22:07.38531526 +0000 UTC m=+0.495879734 container remove 6ee3553a890cd6f14da8c1ebbee2f430d96849a5a54366051c38beb73dc5d50e (image=quay.io/ceph/grafana:9.4.12, name=ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T22:22:07.786 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:07 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T22:22:07.786 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:07 smithi121 systemd[1]: Stopped Ceph grafana.a for 8ccc9cd8-7219-11ef-bceb-c7b262605968. 2024-09-13T22:22:07.786 INFO:journalctl@ceph.grafana.a.smithi121.stdout:Sep 13 22:22:07 smithi121 systemd[1]: ceph-8ccc9cd8-7219-11ef-bceb-c7b262605968@grafana.a.service: Consumed 6.392s CPU time. 2024-09-13T22:22:19.191 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:22:19.225 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:22:19.251 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T22:22:19.256 DEBUG:teuthology.misc:Transferring archived files from smithi114:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231/remote/smithi114/crash 2024-09-13T22:22:19.257 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/crash -- . 2024-09-13T22:22:19.310 INFO:teuthology.orchestra.run.smithi114.stderr:tar: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T22:22:19.311 INFO:teuthology.orchestra.run.smithi114.stderr:tar: Error is not recoverable: exiting now 2024-09-13T22:22:19.312 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231/remote/smithi121/crash 2024-09-13T22:22:19.313 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/crash -- . 2024-09-13T22:22:19.386 INFO:teuthology.orchestra.run.smithi121.stderr:tar: /var/lib/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T22:22:19.387 INFO:teuthology.orchestra.run.smithi121.stderr:tar: Error is not recoverable: exiting now 2024-09-13T22:22:19.388 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T22:22:19.388 DEBUG:teuthology.orchestra.run.smithi114:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | egrep -v CEPHADM_FAILED_DAEMON | egrep -v CEPHADM_AGENT_DOWN | head -n 1 2024-09-13T22:22:19.424 INFO:tasks.cephadm:Compressing logs... 2024-09-13T22:22:19.424 DEBUG:teuthology.orchestra.run.smithi114:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T22:22:19.467 DEBUG:teuthology.orchestra.run.smithi121:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T22:22:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T22:22:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:: No such file or directory 2024-09-13T22:22:19.493 INFO:teuthology.orchestra.run.smithi121.stderr:gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T22:22:19.493 INFO:teuthology.orchestra.run.smithi121.stderr:gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T22:22:19.493 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.7.log: /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T22:22:19.493 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi121.ihcnui.log 2024-09-13T22:22:19.495 INFO:teuthology.orchestra.run.smithi121.stderr:gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T22:22:19.495 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.audit.log: /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi121.ihcnui.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T22:22:19.495 INFO:teuthology.orchestra.run.smithi121.stderr: 62.4% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi121.ihcnui.log.gz 2024-09-13T22:22:19.495 INFO:teuthology.orchestra.run.smithi121.stderr:gzip/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.cephadm.log: -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T22:22:19.496 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.log 2024-09-13T22:22:19.498 INFO:teuthology.orchestra.run.smithi114.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T22:22:19.498 INFO:teuthology.orchestra.run.smithi114.stderr:: No such file or directory 2024-09-13T22:22:19.498 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/tcmu-runner.log 2024-09-13T22:22:19.498 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T22:22:19.498 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/tcmu-runner.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T22:22:19.499 INFO:teuthology.orchestra.run.smithi114.stderr: 87.2% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/tcmu-runner.log.gz 2024-09-13T22:22:19.499 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T22:22:19.499 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi114.vrzgda.log 2024-09-13T22:22:19.499 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-13T22:22:19.499 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi114.vrzgda.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T22:22:19.499 INFO:teuthology.orchestra.run.smithi114.stderr: 62.5% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi114.vrzgda.log.gz 2024-09-13T22:22:19.499 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-13T22:22:19.500 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T22:22:19.500 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-13T22:22:19.500 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.5.log: /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-13T22:22:19.501 INFO:teuthology.orchestra.run.smithi121.stderr: 96.4% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T22:22:19.501 INFO:teuthology.orchestra.run.smithi121.stderr: 89.3% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T22:22:19.501 INFO:teuthology.orchestra.run.smithi121.stderr:gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-13T22:22:19.501 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T22:22:19.501 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.b.log: 91.3% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T22:22:19.505 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.2.log: 97.5% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T22:22:19.505 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.log 2024-09-13T22:22:19.507 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mgr.y.log: 94.7% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T22:22:19.507 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T22:22:19.514 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.log: 94.0%gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T22:22:19.523 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.3.log: -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T22:22:19.556 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/cephadm.log: 92.3% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T22:22:19.558 INFO:teuthology.orchestra.run.smithi121.stderr:/var/log/ceph/cephadm.log: 92.4% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T22:22:19.568 INFO:teuthology.orchestra.run.smithi121.stderr: 91.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T22:22:19.578 INFO:teuthology.orchestra.run.smithi114.stderr: 91.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T22:22:19.645 INFO:teuthology.orchestra.run.smithi121.stderr: 88.6% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-13T22:22:20.048 INFO:teuthology.orchestra.run.smithi114.stderr: 88.3% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-13T22:22:20.069 INFO:teuthology.orchestra.run.smithi121.stderr: 92.4% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-13T22:22:20.229 INFO:teuthology.orchestra.run.smithi114.stderr: 92.4% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-13T22:22:20.685 INFO:teuthology.orchestra.run.smithi121.stderr: 92.8% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T22:22:20.729 INFO:teuthology.orchestra.run.smithi114.stderr: 93.1% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T22:22:20.735 INFO:teuthology.orchestra.run.smithi121.stderr: 93.7% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T22:22:20.848 INFO:teuthology.orchestra.run.smithi114.stderr: 91.4% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-13T22:22:20.908 INFO:teuthology.orchestra.run.smithi121.stderr: 93.4% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T22:22:20.910 INFO:teuthology.orchestra.run.smithi114.stderr: 93.3% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T22:22:21.003 INFO:teuthology.orchestra.run.smithi121.stderr: 93.1% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T22:22:21.005 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-09-13T22:22:21.005 INFO:teuthology.orchestra.run.smithi121.stderr:real 0m1.523s 2024-09-13T22:22:21.005 INFO:teuthology.orchestra.run.smithi121.stderr:user 0m5.988s 2024-09-13T22:22:21.005 INFO:teuthology.orchestra.run.smithi121.stderr:sys 0m0.245s 2024-09-13T22:22:21.104 INFO:teuthology.orchestra.run.smithi114.stderr: 93.2% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T22:22:21.249 INFO:teuthology.orchestra.run.smithi114.stderr: 93.2% -- replaced with /var/log/ceph/8ccc9cd8-7219-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T22:22:21.250 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-09-13T22:22:21.250 INFO:teuthology.orchestra.run.smithi114.stderr:real 0m1.769s 2024-09-13T22:22:21.251 INFO:teuthology.orchestra.run.smithi114.stderr:user 0m8.394s 2024-09-13T22:22:21.251 INFO:teuthology.orchestra.run.smithi114.stderr:sys 0m0.349s 2024-09-13T22:22:21.251 INFO:tasks.cephadm:Archiving logs... 2024-09-13T22:22:21.251 DEBUG:teuthology.misc:Transferring archived files from smithi114:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231/remote/smithi114/log 2024-09-13T22:22:21.252 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T22:22:22.364 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231/remote/smithi121/log 2024-09-13T22:22:22.365 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T22:22:23.227 INFO:tasks.cephadm:Removing cluster... 2024-09-13T22:22:23.227 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 --force 2024-09-13T22:22:23.517 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 --force 2024-09-13T22:22:23.796 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T22:22:23.796 DEBUG:teuthology.orchestra.run.smithi114:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T22:22:23.812 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T22:22:23.828 INFO:tasks.cephadm:Teardown complete 2024-09-13T22:22:23.828 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T22:22:23.837 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T22:22:23.838 DEBUG:teuthology.orchestra.run.smithi114:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:22:23.856 DEBUG:teuthology.orchestra.run.smithi121:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:22:23.867 INFO:teuthology.orchestra.run.smithi114.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:22:23.872 INFO:teuthology.orchestra.run.smithi114.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:22:23.872 INFO:teuthology.orchestra.run.smithi114.stdout:=============================================================================== 2024-09-13T22:22:23.873 INFO:teuthology.orchestra.run.smithi114.stdout:^* hv01.front.sepia.ceph.com 3 7 377 49 -176us[ -175us] +/- 37ms 2024-09-13T22:22:23.873 INFO:teuthology.orchestra.run.smithi114.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 50 +364us[ +365us] +/- 83ms 2024-09-13T22:22:23.873 INFO:teuthology.orchestra.run.smithi114.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 -138us[ -136us] +/- 67ms 2024-09-13T22:22:23.873 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:22:23.882 INFO:teuthology.orchestra.run.smithi121.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:22:23.886 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:22:23.886 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2024-09-13T22:22:23.886 INFO:teuthology.orchestra.run.smithi121.stdout:^* hv01.front.sepia.ceph.com 3 6 377 51 -131us[ -134us] +/- 37ms 2024-09-13T22:22:23.887 INFO:teuthology.orchestra.run.smithi121.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 50 +381us[ +381us] +/- 83ms 2024-09-13T22:22:23.887 INFO:teuthology.orchestra.run.smithi121.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 50 -82us[ -82us] +/- 67ms 2024-09-13T22:22:23.887 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:22:23.887 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T22:22:23.925 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T22:22:23.925 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T22:22:23.934 DEBUG:teuthology.orchestra.run.smithi114:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T22:22:24.022 DEBUG:teuthology.orchestra.run.smithi121:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T22:22:24.101 DEBUG:teuthology.orchestra.run.smithi114:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T22:22:24.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:22:24.130 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-13T22:22:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:22:24.159 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T22:22:24.167 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T22:22:24.177 INFO:teuthology.task.internal:Duration was 2417.002217 seconds 2024-09-13T22:22:24.177 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T22:22:24.186 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T22:22:24.187 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T22:22:24.189 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T22:22:24.225 INFO:teuthology.orchestra.run.smithi114.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:22:24.237 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:22:24.616 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T22:22:24.616 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi114.front.sepia.ceph.com 2024-09-13T22:22:24.616 DEBUG:teuthology.orchestra.run.smithi114:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T22:22:24.676 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi121.front.sepia.ceph.com 2024-09-13T22:22:24.677 DEBUG:teuthology.orchestra.run.smithi121:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T22:22:24.704 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T22:22:24.704 DEBUG:teuthology.orchestra.run.smithi114:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T22:22:24.719 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T22:22:24.775 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T22:22:24.775 DEBUG:teuthology.orchestra.run.smithi114:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T22:22:24.786 DEBUG:teuthology.orchestra.run.smithi121:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T22:22:25.514 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T22:22:25.523 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T22:22:25.524 DEBUG:teuthology.orchestra.run.smithi114:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T22:22:25.548 DEBUG:teuthology.orchestra.run.smithi121:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T22:22:25.575 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T22:22:25.584 DEBUG:teuthology.orchestra.run.smithi114:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:22:25.590 DEBUG:teuthology.orchestra.run.smithi121:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:22:25.611 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern = core 2024-09-13T22:22:25.639 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = core 2024-09-13T22:22:25.661 DEBUG:teuthology.orchestra.run.smithi114:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:22:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:22:25.687 DEBUG:teuthology.orchestra.run.smithi121:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:22:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:22:25.717 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T22:22:25.728 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T22:22:25.730 DEBUG:teuthology.misc:Transferring archived files from smithi114:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231/remote/smithi114 2024-09-13T22:22:25.730 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T22:22:25.791 DEBUG:teuthology.misc:Transferring archived files from smithi121:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904231/remote/smithi121 2024-09-13T22:22:25.792 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T22:22:25.837 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T22:22:25.837 DEBUG:teuthology.orchestra.run.smithi114:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T22:22:25.840 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T22:22:25.883 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T22:22:25.893 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T22:22:25.893 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T22:22:25.902 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T22:22:25.902 DEBUG:teuthology.orchestra.run.smithi114:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T22:22:25.904 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T22:22:25.917 INFO:teuthology.orchestra.run.smithi114.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 22:22 /home/ubuntu/cephtest 2024-09-13T22:22:25.937 INFO:teuthology.orchestra.run.smithi121.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 22:22 /home/ubuntu/cephtest 2024-09-13T22:22:25.939 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T22:22:25.949 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T22:22:25.969 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} duration: 2417.0022172927856 failure_reason: 'Command failed on smithi114 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8ccc9cd8-7219-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c \''ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e \''"\''"\''.up_to_date | length == 7\''"\''"\''\''''' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=316eac3955314b3db4f1fb5429dce82f status: fail success: false 2024-09-13T22:22:25.970 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T22:22:26.076 INFO:teuthology.run:FAIL